```python
import numpy as np
import math
from sympy import *
```
General formula for computing error:
error^2 = sum of [(partial derivative of f in respect to var)^2*(error of var)^2 for each var]
# Derivation of error for basic operators

```python
a,b, a_error, b_error = symbols ('a b a_error b_error')
```

```python
print("sum")
expr_sum = a+b
error_sum = sqrt(diff(expr_sum, a)**2 * a_error**2 + diff(expr_sum, b)**2 * b_error**2)
error_sum
```

    sum

$\displaystyle \sqrt{a_{error}^{2} + b_{error}^{2}}$

```python
print("subtraction")
expr_subtraction = a-b
error_subtraction = sqrt(diff(expr_subtraction, a)**2 * a_error**2 + diff(expr_subtraction, b)**2 * b_error**2)
error_subtraction
```

    subtraction

$\displaystyle \sqrt{a_{error}^{2} + b_{error}^{2}}$

```python
print("multiplication")
expr_multiplication = a*b
error_multiplication = sqrt(diff(expr_multiplication, a)**2 * a_error**2 + diff(expr_multiplication, b)**2 * b_error**2)
error_multiplication
```

    multiplication

$\displaystyle \sqrt{a^{2} b_{error}^{2} + a_{error}^{2} b^{2}}$

```python
print("division")
expr_division = a/b
error_division = sqrt(diff(expr_division, a)**2 * a_error**2 + diff(expr_division, b)**2 * b_error**2)
error_division
```

    division

$\displaystyle \sqrt{\frac{a^{2} b_{error}^{2}}{b^{4}} + \frac{a_{error}^{2}}{b^{2}}}$

```python
print("power")
expr_power = a**b
error_power = sqrt(diff(expr_power, a)**2 * a_error**2 + diff(expr_power, b)**2 * b_error**2)
error_power
```

    power

$\displaystyle \sqrt{a^{2 b} b_{error}^{2} \log{\left(a \right)}^{2} + \frac{a^{2 b} a_{error}^{2} b^{2}}{a^{2}}}$

```python
print("log")
expr_log = log(a,b)
error_log = sqrt(diff(expr_log, a)**2 * a_error**2 + diff(expr_log, b)**2 * b_error**2)
error_log
```

    log

$\displaystyle \sqrt{\frac{b_{error}^{2} \log{\left(a \right)}^{2}}{b^{2} \log{\left(b \right)}^{4}} + \frac{a_{error}^{2}}{a^{2} \log{\left(b \right)}^{2}}}$

```python
print("sin")
expr_sin = sin(a)
error_sin = sqrt(diff(expr_sin, a)**2 * a_error**2)
error_sin
```

    sin

$\displaystyle \sqrt{a_{error}^{2} \cos^{2}{\left(a \right)}}$

```python
print("cos")
expr_cos = cos(a)
error_cos = sqrt(diff(expr_cos, a)**2 * a_error**2)
error_cos
```

    cos

$\displaystyle \sqrt{a_{error}^{2} \sin^{2}{\left(a \right)}}$

```python
print("tan")
expr_tan = tan(a)
error_tan = sqrt(diff(expr_tan, a)**2 * a_error**2)
error_tan
```

    tan

$\displaystyle \sqrt{a_{error}^{2} \left(\tan^{2}{\left(a \right)} + 1\right)^{2}}$

# Derivation of more complex formulas

```python
print("acos")
exp = acos(a)
error = sqrt(diff(exp, a)**2 * a_error**2)
error
```

    acos

$\displaystyle \sqrt{\frac{a_{error}^{2}}{1 - a^{2}}}$

```python
print("acosh")
exp = acosh(a)
error = sqrt(diff(exp, a)**2 * a_error**2)
error
```

    acosh

$\displaystyle \sqrt{\frac{a_{error}^{2}}{a^{2} - 1}}$

```python
print("acot")
exp = acot(a)
error = sqrt(diff(exp, a)**2 * a_error**2)
error
```

    acot

$\displaystyle \sqrt{\frac{a_{error}^{2}}{\left(a^{2} + 1\right)^{2}}}$

```python
print("acoth")
exp = acoth(a)
error = sqrt(diff(exp, a)**2 * a_error**2)
error
```

    acoth

$\displaystyle \sqrt{\frac{a_{error}^{2}}{\left(1 - a^{2}\right)^{2}}}$

```python
print("asin")
exp = asin(a)
error = sqrt(diff(exp, a)**2 * a_error**2)
error
```

    asin

$\displaystyle \sqrt{\frac{a_{error}^{2}}{1 - a^{2}}}$

```python
print("asin")
exp = asinh(a)
error = sqrt(diff(exp, a)**2 * a_error**2)
error
```

    asin

$\displaystyle \sqrt{\frac{a_{error}^{2}}{a^{2} + 1}}$

```python
print("atan")
exp = atan(a)
error = sqrt(diff(exp, a)**2 * a_error**2)
error
```

    atan

$\displaystyle \sqrt{\frac{a_{error}^{2}}{\left(a^{2} + 1\right)^{2}}}$

```python
print("atanh")
exp = atanh(a)
error = sqrt(diff(exp, a)**2 * a_error**2)
error
```

    atanh

$\displaystyle \sqrt{\frac{a_{error}^{2}}{\left(1 - a^{2}\right)^{2}}}$

```python
print("cosh")
exp = cosh(a)
error = sqrt(diff(exp, a)**2 * a_error**2)
error
```

    cosh

$\displaystyle \sqrt{a_{error}^{2} \sinh^{2}{\left(a \right)}}$

```python
print("cot")
exp = cot(a)
error = sqrt(diff(exp, a)**2 * a_error**2)
error
```

    cot

$\displaystyle \sqrt{a_{error}^{2} \left(- \cot^{2}{\left(a \right)} - 1\right)^{2}}$

```python
print("coth")
exp = coth(a)
error = sqrt(diff(exp, a)**2 * a_error**2)
error
```

    coth

$\displaystyle \sqrt{\frac{a_{error}^{2}}{\sinh^{4}{\left(a \right)}}}$

```python
print("csc")
exp = csc(a)
error = sqrt(diff(exp, a)**2 * a_error**2)
error
```

    csc

$\displaystyle \sqrt{a_{error}^{2} \cot^{2}{\left(a \right)} \csc^{2}{\left(a \right)}}$

```python
print("csch")
exp = csch(a)
error = sqrt(diff(exp, a)**2 * a_error**2)
error
```

    csch

$\displaystyle \sqrt{a_{error}^{2} \coth^{2}{\left(a \right)} \operatorname{csch}^{2}{\left(a \right)}}$

```python
print("factorial")
exp = factorial(a)
error = sqrt(diff(exp, a)**2 * a_error**2)
error
```

    factorial

$\displaystyle \sqrt{a_{error}^{2} \Gamma^{2}\left(a + 1\right) \operatorname{polygamma}^{2}{\left(0,a + 1 \right)}}$

```python
print("sec")
exp = sec(a)
error = sqrt(diff(exp, a)**2 * a_error**2)
error
```

    sec

$\displaystyle \sqrt{a_{error}^{2} \tan^{2}{\left(a \right)} \sec^{2}{\left(a \right)}}$

```python
print("sec")
exp = sech(a)
error = sqrt(diff(exp, a)**2 * a_error**2)
error
```

    sec

$\displaystyle \sqrt{a_{error}^{2} \tanh^{2}{\left(a \right)} \operatorname{sech}^{2}{\left(a \right)}}$

```python
print("sinh")
exp = sinh(a)
error = sqrt(diff(exp, a)**2 * a_error**2)
error
```

    sinh

$\displaystyle \sqrt{a_{error}^{2} \cosh^{2}{\left(a \right)}}$

```python
print("tanh")
exp = tanh(a)
error = sqrt(diff(exp, a)**2 * a_error**2)
error
```

    tanh

$\displaystyle \sqrt{a_{error}^{2} \left(1 - \tanh^{2}{\left(a \right)}\right)^{2}}$

```python

```

```python

```

# Creating functions

```python
def sum_error(a, b, a_error, b_error):
    return np.sqrt(a_error**2 + b_error**2)

def subtraction_error(a, b, a_error, b_error):
    return sum_error(a, b, a_error, b_error)

def multiplication_error(a, b, a_error, b_error):
    return np.sqrt((a**2 * b_error**2) + (b**2 * a_error **2))

def division_error(a, b, a_error, b_error):
    return np.sqrt(((a**2 * b_error**2)/ (b**4)) + (a_error**2/b**2))

def power_error(a, b, a_error, b_error):
    return np.sqrt((a**(2*b) * b_error**2 * np.log(a)**2) + ((a**(2*b) * a_error**2 * b**2)/a**2))

def log_error(a, b, a_error, b_error):
    return np.sqrt((b_error**2 * np.log(a)**2)/(b**2 * np.log(b)**4) + a_error**2/(a**2 * np.log(b)**2))

def sin_error(a, b, a_error, b_error):
    return np.sqrt(a_error**2 * np.cos(a)**2)

def cos_error(a, b, a_error, b_error):
    return np.sqrt(a_error**2 * np.sin(a)**2)

def tan_error(a, b, a_error, b_error):
    return np.sqrt(a_error**2 * (np.tan(a)**2 + 1)**2)
```

# Testing of error formulas for basic operators

```python
values = {a: 10, a_error:5, b:8, b_error:3}
```

Derived errors

```python
print("sum =", error_sum.evalf(subs=values))
print("subtraction =", error_subtraction.evalf(subs=values))
print("multiplication =", error_multiplication.evalf(subs=values))
print("division =", error_division.evalf(subs=values))
print("power =", error_power.evalf(subs=values))
print("log =", error_log.evalf(subs=values))
print("sin =", error_sin.evalf(subs=values))
print("cos =", error_cos.evalf(subs=values))
print("tan =", error_tan.evalf(subs=values))
```

    sum = 5.83095189484530
    subtraction = 5.83095189484530
    multiplication = 50.0000000000000
    division = 0.781250000000000
    power = 798229810.232026
    log = 0.312556216389312
    sin = 4.19535764538226
    cos = 2.72010555444685
    tan = 7.10185881291716

Errors from functions

```python
a = 10
b = 8
a_error = 5
b_error = 3
```

```python
print("sum =", sum_error(a, b, a_error, b_error))
print("subtraction =", subtraction_error(a, b, a_error, b_error))
print("multiplication =", multiplication_error(a, b, a_error, b_error))
print("division =", division_error(a, b, a_error, b_error))
print("power =", power_error(a, b, a_error, b_error))
print("log =", log_error(a, b, a_error, b_error))
print("sin =", sin_error(a, b, a_error, b_error))
print("cos =", cos_error(a, b, a_error, b_error))
print("tan =", tan_error(a, b, a_error, b_error))
```

    sum = 5.830951894845301
    subtraction = 5.830951894845301
    multiplication = 50.0
    division = 0.78125
    power = 798229810.2320257
    log = 0.3125562163893125
    sin = 4.195357645382262
    cos = 2.7201055544468495
    tan = 7.101858812917159

```python

```


# Cadenas de Markov

Las cadenas de Markov se han vuelto un tema de gran desarrollo y aplicación en muy distintos ambitos científicos. Esto debido a que se trata de un proceso estocástico (no determinista, i.e. probabilista) que cumple con una gran propiedad a la cual se le conoce como *propiedad de Markov*. __Esta es que el proceso *carece de memoria*__.

Al usar las CM (Cadenas de Markov), en general uno trabajo con *conjuntos de estados*. Tal cual lo hicimos en el Notebook pasado. La **carencia de memoria** se refiere a que un conjunto de estados al tiempo $t$,  $\mathbb{S}_{t}$ depende *únicamente* del conjunto de estados en el paso de tiempo anterior, i.e. $\mathbb{S}_{t-1}$.

De hecho (¡oh casualidad!), los caminantes aleatorios son un tipo particular de CM, ya que la posición al tiempo $t+1$ en una caminata depende sólo de la posición al tiempo $t$ (no de posiciones a tiempos anteriores).

Supongamos ahora que $\mu_n$ es el estado del proceso al tiempo $n$. Por ahora nos interesaremos en procesos de Markov tal que $P(\mu_{n+1} = j \, | \, \mu_n = i) = p_{ij}$, independiente del tiempo. Lo que tenemos aquí es una *probabilidad condicional* la cual nos dice la probabilidad de que el sistema pase del estado $i$ al estado $j$.

Generalizando un poco, podemos llegar fácilmente a que $p_{ij}$ son en realidad entradas de una matriz -cuya importancia es *muy* grande-.

## Matriz de transición

Empecemos con ejemplo para ilustrar una matriz de transición.

Sea un caminante aleatorio que se mueve en una arreglo de 6 celdas, $X = 0, \dotsc, 5$ con fronteras reflejantes y con una probabilidad $p$ de dar un paso a la derecha, $q$ de dar el paso a la izquierda y $r$ de quedarse en el mismo lugar.

No es dificil llegar a que la matriz de transición es:

$$
(p_{ij}) =
\begin{pmatrix}
r + q & p & 0 & 0 & 0 & 0 \\
    q & r & p & 0 & 0 & 0 \\
    0 & q & r & p & 0 & 0 \\
    0 & 0 & q & r & p & 0 \\
    0 & 0 & 0 & q & r & p \\
    0 & 0 & 0 & 0 & q & r + p
\end{pmatrix}
$$

De aquí notamos una primera propiedad muy importante que cualquier matriz de transición ha de cumplir. Si sumamos los elementos de cada fila, hemos entonces de encontrar que el resultado es igual a $1$. Dicho de otro modo:

$$ \sum_{j}p_{ij}=1$$

También podemos notar otra propiedad de este caso, y que tomaremos como general para nuestros propósitos. la matriz de transición $\mathbf{P}$ es **invariante en el tiempo**.

Hagamos un segundo ejemplo. Supongamos una cadena que contiene 4 estados. Desde cada estado se puede saltar a cualquier otro estado con la misma probabilidad.

La matriz de transición en este caso será

$$
(p_{ij}) =
\begin{pmatrix}
r & p & p & p \\
p & r & p & p \\
p & p & r & p \\
p & p & p & r
\end{pmatrix}
$$

De aquí podemos llegar a otra propiedad sobre los elementos de la matriz de transición: $r \geq 0$. Esto podría ser evidente, pero tiene consecuencias importantes.

Evidentemente también para cualquier $i, \ j$: $0 \leq p_{ij} \leq 1$.

**[1]** Como primer ejercicio, el lector tendrá que hacer un `kernel` en el cual se calcule un proceso a partir de una condición inicial, con una matriz de transición y una duración arbitraria.

Consideramos que para este ejercicio, el lector tendrá ya todas las herramientas, sino en la cabeza, en todos los notebooks anteriores.

## Enumeración exacta

Regresemos al método de enumeración exacta para cambiar un poco la perspectiva del problema y enfocarnos más en la *distribución de probabilidad*.

Supongamos que el estado empieza con una distribución $\mu_0 = \{ P_{0}(i) : i = 0, \dotsc, L \}$.

La distribución al paso $t=1$ estará dada por
$$ \mu_{1} = (p_{ij}) \ \mu_{0} $$

¿Qué pasará en el paso $t=2$? Tendremos de nuevo que $\mu_{2} = (p_{ij}) \ \mu_{1}$. Sin embargo según la ecuación siguiente eso implica entonces que

$$\mu_{2} = (p_{ij})^2 \ \mu_0$$

El paso siguiente sería saltar a $t = n$, cuyo resultado es:

$$\mu_{n} = (p_{ij})^n \ \mu_0$$

**[2]** Implementa estas deducciones al código ya escrito y simula de nuevo el proceso.

**[3]** ¿Qué pasa con $\mu_t$ cuando $t \to +\infty$?

## Ergodicidad y balance

En la pregunta pasada se pidió buscar el comportamiento de $\mu_t$ cuando $t \to +\infty$. De esta manera se logró observar como $\mu_t$ convergía hacia una cierta distribución estable. Esta es una propiedad de suma profundidad, y se da únicamente con ciertas caracteristicas de las cuales hablaremos más adelante.

A esta distribución a la que se converge se le llama *distribución estacionaria* y la denotaremos como $\mu_{\infty}$.  Una derivación un poco más formal sería la siguiente:

Puesto que $\mu_t = (p_{ij})^t \ \mu_{0}$, entonces

$$
\begin{align}
\lim_{t \to \infty}\mu_{t}=\lim_{t \to \infty}(p_{ij})^t \ \mu_{0}
\end{align}
$$

Sin embargo $$\lim_{t \to \infty} (p_{ij})^t \ \mu_{0} = (p_{ij}) \lim_{t \to \infty} \mu_t$$ Y al hacer $$\lim_{t \to \infty} \mu_t = \mu_{\infty}$$ obtenemos entonces que

$$ \mu_{\infty} = (p_{ij}) \ \mu_{\infty} \ \ \ (1)$$

Esta última ecuación nos indica una de las propiedades más importantes de la *distribución estacionaria*:
$\mu_{\infty}$ es un __*eigenvector*__ de $(p_{ij})$ cuyo __*eigenvalor*__ $\lambda = 1$.

Otra información importante obtenida de (1) es que una vez que la *distribución estacionaria* es alcanzada, esta ya no cambiará (de ahí su nombre). Es decir que si tomamos como condición inicial $\mu_{\infty}$, la distribución será la misma en cualquier paso de tiempo.

Se puede demostrar (ver las referencias, especialmente el libro de Sethna) que esta distribución existe y es única si la cadena cumple con dos condiciones: es *irreducible* y *aperiódica*

La segunda condición se refiere a que en el proceso estocástico no existen órbitas en las cuales la cadena pueda caer.

La condición de irreducibilidad se traduce como el hecho de que dada una cadena y sus estados, uno puede llegar de un estado dado a cualquier otro con un número finito de pasos. Algo así como que no hay camino que no pueda ser cruzado en un número finito de pasos.

A la suma de estas dos condiciones se le conoce como **ergodicidad**.

El hecho de que el proceso estocástico no sea *aperiódico* puede lograrse pidiendo la condición de __*balance*__. Con ella obtenemos que

$$ \mu \mathbf{P}_{\mu \nu} = \nu \mathbf{P}_{\nu \mu} $$

donde $\mu$ y $\nu$ son dos estados y $\mathbf{P}_{\mu \nu}$ y $\mathbf{P}_{\nu \mu}$ es la misma matriz de transición (el sub índice sólo denota el sentido de la transición. Por ejemplo $\mathbf{P}_{\mu \nu}$ denota la transición $\mu \to \nu$)

Con esta condición podemos decir que la probabilidad de ir de un estado a otro es la misma que hacer el camino de regreso, y por lo tanto los dos caminos son en promedio igual de frecuentes. Se puede demostrar que con este comportamiento las órbitas desaparecen.

Para profundizar más sobre la formalidad y propiedades recomendamos leer el libro de Newman & Barkema listado en las referencias.

**[4]** Dada una matriz de transición, calcula la distribución estacionaria relacionada a la matriz. Puedes buscar distintos métodos para calcularla (tal vez encuentres algo en la librería cuBLAS).

**[5]** ¿Qué tan rápido converge una cadena de Markov hacia su distribución estacionaria?

## Conclusiones hasta el momento

Esperamos que hasta este momento el lector no sólo se sienta ya cómodo con PyCUDA, sino que también sus habilidades a la hora de escribir código tanto en PyCUDA como en CUDA C hayan aumentado.

Este es el último notebook "introductorio" a algunas herramientas y tópicos de física estadística y a partir del siguiente notebook empezaremos a introducir el *modelo de Ising*.

Este modelo tiene grandes ventajas, como profundizar en algunos temas de física estadística, pero sobre todo el conocer y aprender los métodos MCMC (Markov Chain Monte Carlo) y el método de Metropolis-Hastings.

¡Así que con grandes ansias los esperamos en los siguiente notebooks!

## Referencias

+ Wiki de [Markov Chain](https://en.wikipedia.org/wiki/Markov_chain)
+ Newman & Barkema, Monte Carlo Methods in Statistical Physics
+ [Sethna, Statistical Mechanics: Entropy, Order Parameters, and Complexity](http://pages.physics.cornell.edu/~sethna/StatMech/EntropyOrderParametersComplexity.pdf)
+ Documentación de [cuBLAS](http://docs.nvidia.com/cuda/cublas/)

```python

```

Parte 2 - PyCUDA y aplicaciones/05 - Cadenas de Markov.ipynb


# Courbes de Bézier

**TODO**
* https://fr.wikipedia.org/wiki/Courbe_de_B%C3%A9zier#Courbes_de_B.C3.A9zier_cubiques
* https://gist.github.com/Juanlu001/7284462
* https://www.johndcook.com/blog/2009/12/21/bezier-basics/
* https://stackoverflow.com/questions/12643079/b%C3%A9zier-curve-fitting-with-scipy
* Definition using the https://en.wikipedia.org/wiki/Bernstein_polynomial

```python
%matplotlib inline

import ipywidgets
from ipywidgets import interact
```

## Courbe de Bézier quadratique (degré 2)

### Écriture paramétrique

La forme *paramétrique* d'une courbe de Bézier quadratique s'écrit comme une combinaison affine de ses 3 points de contrôle $\mathbf{P}_1$, $\mathbf{P}_2$ et $\mathbf{P}_3 \in \mathbb{R}^2$:
$$
\begin{align}
f: & \mathbb{R} \to \mathbb{R}^2 \\
   & \mathbf{f}(t) \mapsto
       \mathbf{P_1} (1-t)^2
       + 2\mathbf{P_2} t (1-t)
       + \mathbf{P_3} t^2
\end{align}
$$
pour $0 \leq t \leq 1$.

TODO...

## Courbe de Bézier cubique (degré 3)

### Écriture paramétrique

La forme *paramétrique* d'une courbe de Bézier cubique s'écrit comme une combinaison affine de ses 4 points de contrôle $\mathbf{P}_1$, $\mathbf{P}_2$, $\mathbf{P}_3$ et $\mathbf{P}_4 \in \mathbb{R}^2$:
$$
\begin{align}
f: & \mathbb{R} \to \mathbb{R}^2 \\
   & \mathbf{f}(t) \mapsto \mathbf{P}_1 (1-t)^3 + 3 \mathbf{P}_2 t(1-t)^2 + 3 \mathbf{P}_3 t^2 (1-t) + \mathbf{P}_4 t^3
\end{align}
$$
pour $0 \leq t \leq 1$.

```python
import numpy as np

p1 = np.array([0, 0])
p2 = np.array([0, 1])
p3 = np.array([1, 1])
p4 = np.array([1, 0])

def f(t):
    return p1 * (1.-t)**3 + 3. * p2 * t * (1.-t)**2 + 3. * p3 * t**2 * (1.-t) + p4 * t**3

plt.plot(*p1, "xr")
plt.plot(*p2, "xr")
plt.plot(*p3, "xr")
plt.plot(*p4, "xr")

offset = np.array([0.02, 0.02])

plt.text(*(p1 + offset), r"$P_1$")
plt.text(*(p2 + offset), r"$P_2$")
plt.text(*(p3 + offset), r"$P_3$")
plt.text(*(p4 + offset), r"$P_4$")

XY = np.array([f(t) for t in np.linspace(0., 1., 100)])    # TODO: improve this

plt.plot(*XY.T)

plt.xlim(-0.1, 1.1)
plt.ylim(-0.1, 1.1)

plt.show()
```

### Écriture récursive

Avantage: plus efficace en terme de calculs

**TODO**:
* https://en.wikipedia.org/wiki/B%C3%A9zier_curve#Recursive_definition
* https://fr.wikipedia.org/wiki/Courbe_de_B%C3%A9zier#Courbes_de_B.C3.A9zier_cubiques

```python
# TODO
```

### Exemple illustré

```python
class CubicBezierCurve:
    def __init__(self, p1, p2, p3, p4):
        self.p1 = p1
        self.p2 = p2
        self.p3 = p3
        self.p4 = p4

        self.color_deg1 = 'gray'
        self.color_deg2 = 'g'
        self.color_deg3 = 'b'

    def __call__(self, t):
        return self.p1 * (1.-t)**3 \
               + 3. * self.p2 * t * (1.-t)**2 \
               + 3. * self.p3 * t**2 * (1.-t) \
               + self.p4 * t**3

    def init_plot(self, ax=None):
        if ax is None:
            fig, ax = plt.subplots()

        ax.plot(*np.array([self.p1, self.p2]).T, marker='x', color=self.color_deg1)
        ax.plot(*np.array([self.p2, self.p3]).T, marker='x', color=self.color_deg1)
        ax.plot(*np.array([self.p3, self.p4]).T, marker='x', color=self.color_deg1)

        offset = np.array([0.02, 0.02])

        ax.text(*(self.p1 + offset), r"$P_1$")
        ax.text(*(self.p2 + offset), r"$P_2$")
        ax.text(*(self.p3 + offset), r"$P_3$")
        ax.text(*(self.p4 + offset), r"$P_4$")

        T = np.linspace(0., 1., 100)
        XY = np.array([self(t) for t in T])    # TODO: improve this

        ax.plot(*XY.T, 'r:', alpha=0.5)

        points = np.array([self.p1, self.p2, self.p3, self.p4])
        ax.set_xlim(points[:,0].min() - 0.1, points[:,0].max() + 0.1)
        ax.set_ylim(points[:,1].min() - 0.1, points[:,1].max() + 0.1)

        return ax

    def plot(self, t, ax=None):
        if ax is None:
            fig, ax = plt.subplots()

        p12 = self.p1 + t * (self.p2 - self.p1)
        p23 = self.p2 + t * (self.p3 - self.p2)
        p34 = self.p3 + t * (self.p4 - self.p3)

        p1223 = p12 + t * (p23 - p12)
        p2334 = p23 + t * (p34 - p23)

        ax.plot(*np.array([p12, p23, p34]).T, marker='.', color=self.color_deg2)
        ax.plot(*np.array([p1223, p2334]).T, marker='.', color=self.color_deg3)

        ax.plot(*self(t).T, 'or')

p1 = np.array([0, 0])
p2 = np.array([0, 1])
p3 = np.array([1, 1])
p4 = np.array([1, 0])
f = CubicBezierCurve(p1, p2, p3, p4)

ax = f.init_plot()
f.plot(0.25, ax)

plt.show();
```

### Exemple interactif

```python
@interact(t=(0., 1., 0.01))
def square(t):
    p1 = np.array([0, 0])
    p2 = np.array([0, 1])
    p3 = np.array([1, 1])
    p4 = np.array([1, 0])
    f = CubicBezierCurve(p1, p2, p3, p4)

    ax = f.init_plot()
    f.plot(t, ax)

    plt.show();
```

## Courbes de Bézier quartique (degré 4)

TODO...


# Finite size scaling

In this notebook we will discuss the use of a finite size scaling ansatz to measure properties of a percolating system. So far we've treated the finite systems as if they were in reality infinte.

We will start by finding the percolation threshold $p_{\Pi = x}$ for which the percolation probability $\Pi(p_{\Pi = x}) = x$. That is, we generate many systems of size $L$ with a specific probability $p$ and compute the average percolation probability. We improve on our choice of $p$ until we reach a point where $\Pi(p) \approx x$. The value of $p$ at this point will be $p_{\Pi = x}$.

```python
import numpy as np
import scipy.optimize
import tqdm
import matplotlib.pyplot as plt
import seaborn as sns
from percolation import compute_percolation_threshold, compute_percolation_probability
import sklearn.linear_model
```

```python
sns.set(color_codes=True)
```

```python
L_list = [25, 50, 100, 200, 400, 800]
M = 100

x_high = 0.8
x_low = 0.3

p_pi_high = []
p_pi_low = []
```

```python
for L in tqdm.tqdm_notebook(L_list):
    p_pi = compute_percolation_threshold(x_high, L, M, p_bounds=(0, 1))
    print(f"(high) p = {p_pi}")
    p_pi_high.append(p_pi)

    p_pi = compute_percolation_threshold(x_low, L, M, p_bounds=(0, 1))
    p_pi_low.append(p_pi)
    print(f"(low) p = {p_pi}")
```

    (high) p = 0.6093978881835938
    (low) p = 0.5481948852539062
    (high) p = 0.5994186401367188
    (low) p = 0.5694656372070312
    (high) p = 0.5993118286132812
    (low) p = 0.5761032104492188
    (high) p = 0.5969009399414062
    (low) p = 0.5837020874023438
    (high) p = 0.5949020385742188
    (low) p = 0.5875167846679688
    (high) p = 0.5938339233398438
    (low) p = 0.5895919799804688

```python
fig = plt.figure(figsize=(14, 10))

plt.plot(L_list, p_pi_high, label=fr"$x = {x_high}$", marker="o")
plt.plot(L_list, p_pi_low, label=fr"$x = {x_low}$", marker="o")
plt.legend(loc="best")
plt.xlabel(r"$L$")
plt.ylabel(r"$p_{\Pi = x}$")
plt.title(r"Plot of the effective percolation threshold as a function of system size $L$")
plt.show()
```

## Scaling as a function of system size

We know that the scaling theory for the percolation threshold can be found from
\begin{align}
    p_{\Pi = \Delta x}
    \equiv
    p_{\Pi = x_1} - p_{\Pi = x_2}
    = \left( C_{x_1} - C_{x_2} \right) L^{-1 / \nu}.
\end{align}
Taking the logarithm on both sides, we can compute an estimate for the exponent $-1 / \nu$.
\begin{align}
    \log\left( p_{\Pi = \Delta x} \right)
    = \log \left( C_{x_1} - C_{x_2} \right)
    - \frac{1}{\nu}\log\left( L \right).
\end{align}

```python
p_diff = np.array(p_pi_high) - np.array(p_pi_low)
L_arr = np.array(L_list)
```

Below we do linear regression to find an estimate for the exponent $\nu$.

```python
log_p_diff = np.log(p_diff)
log_L_arr = np.log(L_arr)

clf = sklearn.linear_model.LinearRegression().fit(
    log_L_arr[:, np.newaxis], log_p_diff[:, np.newaxis]
)

neg_inv_nu = clf.coef_[0, 0]
log_C_diff = clf.intercept_[0]
```

```python
nu = -1 / neg_inv_nu

print(f"nu = {nu}")
```

    nu = 1.3395479342334302

The true value of $\nu$ for $2$ dimensions is $\nu = 4 / 3 \approx 1.3333$.

```python
nu_true = 4 / 3

print(f"Diff in nu from true value: {np.abs(nu_true - nu)}")
```

    Diff in nu from true value: 0.006214600900096956

```python
plt.figure(figsize=(14, 10))

plt.plot(
    log_L_arr,
    log_p_diff,
    lw=2,
    label=r"Data"
)
plt.plot(
    log_L_arr,
    log_C_diff + neg_inv_nu * log_L_arr,
    "--",
    lw=2,
    label=r"Estimated value for $\nu$"
)
plt.plot(
    log_L_arr,
    log_C_diff - 1 / nu_true * log_L_arr,
    "--",
    lw=2,
    label=r"True value for $\nu$"
)
plt.legend(loc="best")
plt.title(r"Scaling of the percolation threshold as function of system size")
plt.xlabel(r"$\log(L)$")
plt.ylabel(r"$\log(p_{\Pi = \Delta x})$")
plt.show()
```

Here we have plotted the computed values for the scaling of the difference between two percolation thresholds, versus the theoretical model using both the estimated value for $\nu$ and the "true" value. We see that the values are pretty close.

## Estimating the critical percolation probability

Using scaling theory from the percolation threshold, we can estimate the critical percolation probability $p_c$ by
\begin{align}
    p_{\Pi = x} = p_c + C_x L^{-1 / \nu},
\end{align}

where $\nu = 4 / 3$ is the exact known value. Again we use linear regression to estimate the coefficient $C_x$ and the intercept, i.e., the critical percolation probability, $p_c$.

```python
p_pi_high_arr = np.array(p_pi_high)
p_pi_low_arr = np.array(p_pi_low)
```

```python
clf_high = sklearn.linear_model.LinearRegression().fit(
    L_arr[:, np.newaxis] ** (-1 / nu_true),
    p_pi_high_arr[:, np.newaxis]
)

C_x_high = clf_high.coef_[0, 0]
p_c_high = clf_high.intercept_[0]

print(f"Estimated p_c for x = {x_high}: {p_c_high:.4f}")
```

    Estimated p_c for x = 0.8: 0.5929

```python
clf_low = sklearn.linear_model.LinearRegression().fit(
    L_arr[:, np.newaxis] ** (-1 / nu_true),
    p_pi_low_arr[:, np.newaxis]
)

C_x_low = clf_low.coef_[0, 0]
p_c_low = clf_low.intercept_[0]

print(f"Estimated p_c for x = {x_low}: {p_c_low:.4f}")
```

    Estimated p_c for x = 0.3: 0.5930

We see that our estimate for the critical percolation probability is quite close the the known "true" value of $p_c=0.59275$.

```python
p_c = 0.59275
```

```python
plt.figure(figsize=(14, 10))

L_nu = L_arr ** (-1 / nu_true)

plt.plot(
    L_nu,
    p_pi_high_arr,
    "-o",
    lw=2,
    label=r"$p_{\Pi = 0.8}$"
)
plt.plot(
    L_nu,
    p_pi_low_arr,
    "-o",
    lw=2,
    label=r"$p_{\Pi = 0.3}$"
)

plt.plot(
    L_nu,
    clf_high.predict(L_nu[:, np.newaxis]).ravel(),
    "--",
    lw=2,
    label=r"$\tilde{p}_{\Pi = 0.8}$"
)
plt.plot(
    L_nu,
    clf_low.predict(L_nu[:, np.newaxis]).ravel(),
    "--",
    lw=2,
    label=r"$\tilde{p}_{\Pi = 0.3}$"
)

plt.legend(loc="best")
plt.xlabel(r"$L^{-1/\nu}$")
plt.ylabel(r"$p$")
plt.title(r"Plot of the scaling of the effective percolation threshold")
plt.show()
```

In the lecture notes we find that we can write the scaling anstatz of the percolation threshold to be
\begin{gather}
    p = p_c + C L^{-1 / \nu}
    \implies
    \left(p - p_c\right) L^{1 / \nu} = C \equiv \Phi^{-1}(x).
\end{gather}
We defined $x$ to be a certain value of $\Pi(p, L)$, that is,
\begin{align}
    x = \Pi(p, L).
\end{align}
Therefore, if we apply the function $\Phi(u)$ on both sides of the former equation, we are left with
\begin{align}
    x = \Pi(p, L) = \Phi\left(
        \left[ p - p_c \right]
        L^{1 / \nu}
    \right).
\end{align}
By plotting $u = \left[ p - p_c \right] L^{1 / \nu}$ versus $\Pi(p, L)$ we find the shape of $\Phi(u)$.

```python
p_arr = np.linspace(0.5, 0.65, 21)
L_new = 2 ** np.arange(4, 9)
M = 100

Pi_arr = np.zeros((len(L_new), len(p_arr)))
```

```python
for i, L in tqdm.tqdm_notebook(enumerate(L_new), total=len(L_new)):
    for j, p in enumerate(p_arr):
        Pi_arr[i, j] = compute_percolation_probability(L, p, M)
```

```python
u = np.zeros(Pi_arr.shape)

for i, L in enumerate(L_new):
    u[i] = (p_arr - p_c) * L ** (1 / nu_true)
```

```python
plt.figure(figsize=(14, 10))

for i in range(len(L_new)):
    plt.plot(u[i], Pi_arr[i], label=fr"$L = {L_new[i]}$")

plt.legend(loc="best")
plt.title(r"Plot of the scaling function for $\Pi(p, L)$")
plt.xlabel(r"$\left[p - p_c\right] L^{1 / \nu}$")
plt.ylabel(r"$\Pi(p, L)$")
plt.show()
```

Here we see how $\Pi(p, L)$ scales as a function of $u = \left[ p - p_c\right] L^{1 / \nu}$. This shape is the same as the function $\Phi(u)$.


# Softmax Classifier笔记

> 2018年8月10日， 罗周杨

支持向量机(SVM)是两个最常用的分类器之一，另一个就是**Softmax classifier**，它和损失函数和SVM不一样。如果说逻辑回归分类器是二分类器，那么softmax就是多分类器。

不同于SVM把输出$f(x_{i};W)$作为分数（未经校准并且可能难以解释），softmax的输出是一个**归一化概率(normalized class probabilities)**。

Softmax使用**交叉熵(cross-entropy loss)** 代替了**hinge loss(即$\max(0,-)$函数）**。

**cross-entropy loss**公式如下：

$$L_{i}=-log(\frac{e^{f_{y_{i}}}} {\sum_{j} e^{f_{j}}})$$

或者写成：

$$L_{i}=-f_{y_{i}}+log\sum_{j}e^{f_{j}}$$

其中$f_{j}$表示第$j$个元素的分数$f$。

和Multiclass SVM classifier一样，对于整个数据集的损失定义为：

\begin{align}
L&=\frac{1}{N}\sum_{j}L_{i}+\lambda R(W) \\
 &=\frac{1}{N}\sum_{j}-log(\frac{e^{f_{y_{i}}}} {\sum_{j} e^{f_{j}}})+\lambda R(W)
\end{align}

## Softmax function

Softmax function的定义如下：

$$f_{j}(z)=\frac{e^{z_{j}}}{\sum_{k}e^{z_{k}}}$$

它对于真正的分数向量$z$压成一个(0,1)区间的值，这些值的和为1。

根据信息论，真实分布$p$和估计分布$q$之间的**交叉熵**定义为：

$$ H(p,q)=-\sum_{x}p(x)logq(x)$$

因此，Softmax分类器就是在最小化估计的类别概率（$p=\frac{e^{f_{y_{i}}}} {\sum_{j} e^{f_{j}}}$）和真实分布之间的交叉熵。

**疑问**：

* **交叉熵和softmax函数的更具体的解释**。

从概率论的角度，式子：

$$
P(y_{i}|x_{i};W)=\frac{e^{f_{y_{i}}}} {\sum_{j} e^{f_{j}}}
$$

可以解释成：正确类别y关于输入x和权重矩阵W的**概率**。

**疑问**:

* **概率论更加具体的解释**。

## SVM和Softmax的区别：

* Softmax提供的是每一个类别的"概率”，打上冒号是因为，这个“概率”直接影响于正则化参数$\lambda$的值。


```python
%pylab inline
```

    Populating the interactive namespace from numpy and matplotlib

```python
N = 11
h = 1/(N-1)
x = linspace(0,1,N)
```

```python
f = ones((N,))
```

```python
A = zeros((N,N))
for i in range(1,N-1):
    A[i, i-1] = A[i, i+1] = -1
    A[i,i] = 2
A[0,0] = A[-1,-1] = 1
f[0] = f[-1] = 0

A = A/h**2
```

```python
A, f
```

    (array([[ 100.,    0.,    0.,    0.,    0.,    0.,    0.,    0.,    0.,
                0.,    0.],
            [-100.,  200., -100.,    0.,    0.,    0.,    0.,    0.,    0.,
                0.,    0.],
            [   0., -100.,  200., -100.,    0.,    0.,    0.,    0.,    0.,
                0.,    0.],
            [   0.,    0., -100.,  200., -100.,    0.,    0.,    0.,    0.,
                0.,    0.],
            [   0.,    0.,    0., -100.,  200., -100.,    0.,    0.,    0.,
                0.,    0.],
            [   0.,    0.,    0.,    0., -100.,  200., -100.,    0.,    0.,
                0.,    0.],
            [   0.,    0.,    0.,    0.,    0., -100.,  200., -100.,    0.,
                0.,    0.],
            [   0.,    0.,    0.,    0.,    0.,    0., -100.,  200., -100.,
                0.,    0.],
            [   0.,    0.,    0.,    0.,    0.,    0.,    0., -100.,  200.,
             -100.,    0.],
            [   0.,    0.,    0.,    0.,    0.,    0.,    0.,    0., -100.,
              200., -100.],
            [   0.,    0.,    0.,    0.,    0.,    0.,    0.,    0.,    0.,
                0.,  100.]]),
     array([0., 1., 1., 1., 1., 1., 1., 1., 1., 1., 0.]))

```python
u = linalg.solve(A, f)
```

```python
plot(x, u, 'o-b')
plot(x, 0.5*(x*(1-x)), 'r')
```

```python
exact = 0.5*(x*(1-x))
error = max(abs(exact-u))
```

```python
error
```

    4.163336342344337e-17

```python
exact = x*(1-x)*(x-.85)
```

```python
import sympy as sym
t = sym.var('x')
exact_t = t*(1-t)*(t-.85)
fsymbol = sym.lambdify(t, -exact_t.diff(t, 2) )

x = linspace(0,1,N)
f = fsymbol(x)
```

```python
x = linspace(0,1,N)
f = fsymbol(x)
f[0] = f[-1] = 0
```

```python
plot(x,f)
```

```python
u = linalg.solve(A, f)
```

```python
plot(x, u, 'ob-')
plot(x, exact)
```

```python
max(abs(u - exact))
```

    3.469446951953614e-17

```python
x = sym.var('x')
h = sym.var('h')
g = sym.Function('g')
```

```python
def cfd_II(x,h,g):
    return (g(x+h)- 2*g(x) + g(x-h))/h**2

def back_fd(x,h,g):
    return (g(x+h)- g(x))/h

def forward_fd(x,h,g):
    return (g(x)- g(x-h))/h

def central_fd(x,h,g):
    return (g(x+h)- g(x-h))/(2*h)
```

```python
sym.series(back_fd(x, h, g), x=h, x0=0, n=2)
```

$\displaystyle \left. \frac{d}{d \xi_{1}} g{\left(\xi_{1} \right)} \right|_{\substack{ \xi_{1}=x }} + \frac{h \left. \frac{d^{2}}{d \xi_{1}^{2}} g{\left(\xi_{1} \right)} \right|_{\substack{ \xi_{1}=x }}}{2} + O\left(h^{2}\right)$

```python
sym.series(forward_fd(x, h, g), x=h, x0=0, n=2)
```

$\displaystyle \left. \frac{d}{d \xi_{1}} g{\left(\xi_{1} \right)} \right|_{\substack{ \xi_{1}=x }} - \frac{h \left. \frac{d^{2}}{d \xi_{1}^{2}} g{\left(\xi_{1} \right)} \right|_{\substack{ \xi_{1}=x }}}{2} + O\left(h^{2}\right)$

```python
sym.series(central_fd(x, h, g), x=h, x0=0, n=3)
```

$\displaystyle \left. \frac{d}{d \xi_{1}} g{\left(\xi_{1} \right)} \right|_{\substack{ \xi_{1}=x }} + \frac{h^{2} \left. \frac{d^{3}}{d \xi_{1}^{3}} g{\left(\xi_{1} \right)} \right|_{\substack{ \xi_{1}=x }}}{6} + O\left(h^{3}\right)$

```python
sym.series(cfd_II(x, h, g), x=h, x0=0, n=5)
```

$\displaystyle \left. \frac{d^{2}}{d \xi_{1}^{2}} g{\left(\xi_{1} \right)} \right|_{\substack{ \xi_{1}=x }} + \frac{h^{2} \left. \frac{d^{4}}{d \xi_{1}^{4}} g{\left(\xi_{1} \right)} \right|_{\substack{ \xi_{1}=x }}}{12} + \frac{h^{4} \left. \frac{d^{6}}{d \xi_{1}^{6}} g{\left(\xi_{1} \right)} \right|_{\substack{ \xi_{1}=x }}}{360} + O\left(h^{5}\right)$

slides/.ipynb_checkpoints/Lecture 10 - LH - LAB - Introduction to PDEs - Finite Differences in 1D-checkpoint.ipynb


# Problem set 1: Solving the consumer problem

In this first problem set, we will take a look at solving the canonical utility maximization problem for the consumer.

* **Problem set structure:** Each problem set consists of tasks and problems. _Tasks_ train you in using specific techniques, while _problems_ train you in solving actual economic problems. Each problem set also contains solutions in hidden cells. *You should really try to solve the tasks and problems on your own before looking at the answers!*

> **Workflow:** You goal should, however, not be to write everything from scratch. Finding similar code from the lectures and adjusting it is completely ok. I rarely begin  completely from scratch, I figure out when I last did something similar and copy in the code to begin with.

* **Multiple solutions:** Within the field of numerical analysis there is often many more than one way of solving a specific problem. So the solution provided is just one example. If you get the same result, but use another approach, that might be just as good (or even better).

* **Extra problems:** Solutions to the extra problems are not provided, but we encourage you to take a look at them if you have the time. You can share your solution with your fellow students following this [guide](https://numeconcopenhagen.netlify.com/guides/snippets/).

# Tasks

## functions

Implement a Python version of this function:

\\[ u(x_1,x_2) = (\alpha x_1^{-\beta} + (1-\alpha) x_2^{-\beta})^{-1/\beta} \\]

```python
# write your code here
```

**Answer:**

```python
def u(x1,x2,alpha=0.5,beta=1):
    return (alpha*x1**(-beta) + (1-alpha)*x2**(-beta))**(-1/beta)
```

## print

```python
x1_vec = [1.05,1.3,2.3,2.5,3.1]
x2_vec = [1.05,1.3,2.3,2.5,3.1]
```

Construct a Python function `print_table(x1_vec,x2_vec)` to print values of `u(x1,x2)` in the table form shown below.

```python
# update this code

def print_table(x1_vec,x2_vec):

    # a. empty text
    text = ''

    # b. top header
    text += f'{"":3s}'
    for j, x2 in enumerate(x2_vec):
       text += f'{j:6d}'
    text += '\n' # line shift

    # c. body
    # missing lines

    # d. print
    print(text)
```

**Answer:**

```python
def print_table(x1_vec,x2_vec):

    # a. empty text
    text = ''

    # b. top header
    text += f'{"":3s}'
    for j, x2 in enumerate(x2_vec):
       text += f'{j:6d}'
    text += '\n' # line shift

    # c. body
    for i,x1 in enumerate(x1_vec):
        if i > 0:
            text += '\n' # line shift
        text += f'{i:3d} ' # left header
        for j, x2 in enumerate(x2_vec):
            text += f'{u(x1,x2):6.3f}'

    # d. print
    print(text)

print_table(x1_vec,x2_vec)
```

            0     1     2     3     4
      0  1.050 1.162 1.442 1.479 1.569
      1  1.162 1.300 1.661 1.711 1.832
      2  1.442 1.661 2.300 2.396 2.641
      3  1.479 1.711 2.396 2.500 2.768
      4  1.569 1.832 2.641 2.768 3.100

## matplotlib

Reproduce the figure below of \\(u(x_1,x_2)\\) using the `meshgrid` function from _numpy_ and the `plot_surface` function from _matplotlib_.

```python
# evaluate utility function
import numpy as np
x1_grid,x2_grid = np.meshgrid(x1_vec,x2_vec,indexing='ij')
u_grid = u(x1_grid,x2_grid)

# import plot modules
%matplotlib inline
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from matplotlib import cm # for colormaps

# write your code here
```

**Answer:**

```python
# a. plot
fig = plt.figure()
ax = fig.add_subplot(1,1,1,projection='3d')
ax.plot_surface(x1_grid,x2_grid,u_grid,cmap=cm.jet)

# b. add labels
ax.set_xlabel('$x_1$')
ax.set_ylabel('$x_2$')
ax.set_zlabel('$utility,u$')

# c. invert xaxis
ax.invert_xaxis()
```

## optimize

Consider the following minimization problem:

\\[\min_x  f(x) = \min_x \sin(x) + 0.05 \cdot x^2 \\]

Solve this problem and illustrate your results.

```python
# update this code

# a. define function
def f(x):
    return 0 # wrong line

# b. solution using a loop
import numpy as np
N = 100
x_vec = np.linspace(-10,10,N)
f_vec = np.empty(N)

f_best = np.inf # initial maximum
x_best = np.nan # not-a-number

for i,x in enumerate(x_vec):
    f_now = f_vec[i] = f(x)
    # missing lines

# c. solution using scipy optmize
from scipy import optimize
x_guess = [0]
# missing line, hint: objective_function = lambda x: ?
# missing line, hint: res = optimize.minimize(?)
# x_best_scipy = res.x[0]
# f_best_scipy = res.fun

# d. print
# missing lines

# e. figure
# missing lines
```

**Answer:**

```python
# a. define function
def f(x):
    return np.sin(x)+0.05*x**2

# b. solution using a loop
import numpy as np
N = 100
x_vec = np.linspace(-10,10,N)
f_vec = np.empty(N)

f_best = np.inf # initial maximum
x_best = np.nan # not-a-number

for i,x in enumerate(x_vec):
    f_now = f_vec[i] = f(x)
    if f_now < f_best:
        x_best = x
        f_best = f_now

# c. solution using scipy optmize
from scipy import optimize
x_guess = [0]
objective_function = lambda x: f(x[0])
res = optimize.minimize(objective_function, x_guess, method='Nelder-Mead')
x_best_scipy = res.x[0]
f_best_scipy = res.fun

# d. print
print(f'best with loop is           {f_best:.8f} at x = {x_best:.8f}')
print(f'best with scipy.optimize is {f_best_scipy:.8f} at x = {x_best_scipy:.8f}')

# e. figure
import matplotlib.pyplot as plt
fig = plt.figure() # dpi = dots-per-inch (resolution)
ax = fig.add_subplot(1,1,1)

ax.plot(x_vec,f_vec,ls='--',lw=2,color='black',label='$f(x)$')
ax.plot(x_best,f_best,ls='',marker='s',color='blue',label='loop')
ax.plot(x_best_scipy,f_best_scipy,ls='',marker='o',
        markersize=10,markerfacecolor='none',
        markeredgecolor='red',label='scipy.optimize')

ax.set_xlabel('$x$')
ax.set_ylabel('$f$')
ax.grid(True)
ax.legend(loc='upper center');
```

# Problem

Consider the following \\(M\\)-good, \\(x=(x_1,x_2,\dots,x_M)\\), **utility maximization problem** with exogenous income \\(I\\), and price-vector \\(p=(p_1,p_2,\dots,p_M)\\),

\\[
\begin{align}
V(p_{1},p_{2},\dots,,p_{M},I) & = \max_{x_{1},x_{2},\dots,x_M} x_{1}^{\alpha_1} x_{2}^{\alpha_2} \dots x_{M}^{\alpha_M} \\
 & \text{s.t.}\\
E = \sum_{i=1}^{M}p_{i}x_{i} & \leq I,\,\,\,p_{1},p_{2},\dots,p_M,I>0\\
x_{1},x_{2},\dots,x_M & \geq 0
\end{align}
\\]

**Problem:** Solve the 5-good utility maximization problem for arbitrary preference parameters, \\( \alpha = (\alpha_1,\alpha_2,\dots,\alpha_5)\\), prices and income. First, with a loop, and then with a numerical optimizer.

You can use the following functions:

```python
def utility_function(x,alpha):
    # ensure you understand what this function is doing

    u = 1
    for x_now,alpha_now in zip(x,alpha):
        u *= np.max(x_now,0)**alpha_now
    return u

def expenditures(x,p):
    # ensure you understand what this function is doing

    E = 0
    for x_now,p_now in zip(x,p):
        E += p_now*x_now
    return E

def print_solution(x,alpha,I,p):
    # you can just use this function

    # a. x values
    text = 'x = ['
    for x_now in x:
        text += f'{x_now:.2f} '
    text += f']\n'

    # b. utility
    u = utility_function(x,alpha)
    text += f'utility = {u:.3f}\n'

    # c. expenditure vs. income
    E =  expenditures(x,p)
    text += f'E = {E:.2f} <= I = {I:.2f}\n'

    # d. expenditure shares
    e = p*x/I
    text += 'expenditure shares = ['
    for e_now in e:
        text += f'{e_now:.2f} '
    text += f']'

    print(text)
```

You can initially use the following parameter choices:

```python
alpha = np.ones(5)/5
p = np.array([1,2,3,4,5])
I = 10
```

Solving with a loop:

```python
# update this code

N = 15 # number of points in each dimension
fac = np.linspace(0,1,N) # vector betweein 0 and 1
x_max = I/p # maximum x so E = I

# missing lines
for x1 in fac:
   for x2 in fac:
        for x3 in fac:
            for x4 in fac:
                for x5 in fac:
                    x = np.array([x1,x2,x3,x4,x5])*x_max
                    E = expenditures(x,p)
                    if E <= I:
                        u_now = utility_function(x,alpha)
                        # misssing lines

# print_solution(x_best,alpha,I,p)
```

> **Extra:** The above code can be written nicer with the ``product`` function from ``itertools``.

Solving with a numerical optimizer:

```python
# update this code

from scipy import optimize

# a. contraint function (negative if violated)
# missing line, hint: constraints = ({'type': 'ineq', 'fun': lambda x: ?})
# missing line, hint: bounds = [(?,?) for p_now in p]

# b. call optimizer
initial_guess = (I/p)/6 # some guess, should be feasible
# missing line, hint: res = optimize.minimize(?,?,method='SLSQP',bounds=bounds,constraints=constraints)

# print(res.message) # check that the solver has terminated correctly

# c. print result
# print_solution(res.x,alpha,I,p)
```

## Solutions using loops

Using **raw loops**:

```python
N = 15 # number of points in each dimension
fac = np.linspace(0,1,N) # vector betweein 0 and 1
x_max = I/p # maximum x so E = I

u_best = -np.inf
x_best = np.empty(5)
for x1 in fac:
   for x2 in fac:
        for x3 in fac:
            for x4 in fac:
                for x5 in fac:
                    x = np.array([x1,x2,x3,x4,x5])*x_max
                    E = expenditures(x,p)
                    if E <= I:
                        u_now = utility_function(x,alpha)
                        if u_now > u_best:
                            x_best = x
                            u_best = u_now

print_solution(x_best,alpha,I,p)
```

    x = [2.14 1.07 0.71 0.36 0.43 ]
    utility = 0.758
    E = 10.00 <= I = 10.00
    expenditure shares = [0.21 0.21 0.21 0.14 0.21 ]

Using **smart itertools loop:**

```python
import itertools as it

N = 15 # number of points in each dimension
fac = np.linspace(0,1,N) # vector betweein 0 and 1
x_max = I/p # maximum x so E = I

x_best = np.empty(5)
u_best = -np.inf
for x in it.product(fac,fac,fac,fac,fac):
    x *= x_max
    E = expenditures(x,p)
    if E <= I:
        u_now = utility_function(x,alpha)
        if u_now > u_best:
            x_best = x
            u_best = u_now

print_solution(x_best,alpha,I,p)
```

    x = [2.14 1.07 0.71 0.36 0.43 ]
    utility = 0.758
    E = 10.00 <= I = 10.00
    expenditure shares = [0.21 0.21 0.21 0.14 0.21 ]

## Solutions using solvers

```python
from scipy import optimize
```

Solution using a **constrained optimizer:**

```python
# a. contraint function (negative if violated)
constraints = ({'type': 'ineq', 'fun': lambda x:  I-expenditures(x,p)})
bounds = [(0,I/p_now) for p_now in p]

# b. call optimizer
initial_guess = (I/p)/6 # some guess, should be feasible
res = optimize.minimize(
    lambda x: -utility_function(x,alpha),initial_guess,
    method='SLSQP',bounds=bounds,constraints=constraints)

print(res.message) # check that the solver has terminated correctly

# c. print result
print_solution(res.x,alpha,I,p)
```

    Optimization terminated successfully.
    x = [2.00 1.00 0.67 0.50 0.40 ]
    utility = 0.768
    E = 10.00 <= I = 10.00
    expenditure shares = [0.20 0.20 0.20 0.20 0.20 ]

Solution using an **unconstrained optimizer:**

```python
# a. define objective function
def unconstrained_objective(x,alpha,I,p):

    penalty = 0
    E = expenditures(x,p)
    if E >= I:
        ratio = I/E
        x *= ratio # now p*x = I
        penalty = 1000*(E-I)**2

    u = utility_function(x,alpha)
    return -u + penalty
    # note:
    #  "-u" because we are minimizing
    #  "+ penalty" because the minimizer
    #   will then avoid the E > I

# b. call optimizer
initial_guess = (I/p)/6
res = optimize.minimize(
    unconstrained_objective,initial_guess,
    method='Nelder-Mead',args=(alpha,I,p),options={'maxiter':5000},tol=1e-10)

print(res.message)

# c. print result
print_solution(res.x,alpha,I,p)
```

    Optimization terminated successfully.
    x = [2.00 1.00 0.67 0.50 0.40 ]
    utility = 0.768
    E = 10.00 <= I = 10.00
    expenditure shares = [0.20 0.20 0.20 0.20 0.20 ]

# Extra Problems

## Cost minimization

Consider the following 2-good **cost minimziation problem** with required utility \\(u_0\\), and price-vector \\(p=(p_1,p_2)\\),

\\[
\begin{align}
E(p_{1},p_{2},u_0) & = \min_{x_{1},x_{2}} p_1 x_1+p_2 x_2\\
 & \text{s.t.}\\
x_{1}^{\alpha}x_{2}^{1-\alpha} & \geq u_0 \\
x_{1},x_{2} & \geq 0
\end{align}
\\]

**Problem:** Solve the 2-good cost-minimization problem with arbitrary required utility, prices and income. Present your results graphically showing that the optimum is a point, where a budgetline is targent to the indifference curve through $u_0$.

## Classy solution

**Problem:** Implement your solution to the utility maximization problem and/or the cost minimization problem above in a class as seen in Lecture 3.


# Some introduction to entropy

**`Entropy`** (or **`expected surprisal`**) is a measure of either information given by probability or of the chaos present in a system (more or less: *how much of my data is easy to classify ?*).

In the context of `machine learning`, `entropy` is a characteristic of our dataset. The **more entropy**, the **better suited the data is for learning**, the **less entropy**, the more uniform and pure the data is, as such, learning classification on that dataset **will be very prone to overfitting** (learning will lead to a model that will be very good on our learning data, but will fail on `validation` and/or `testing` data -> it will be a model that is specific to some set of data, and not `generalized` and `usable in other contexts`).

### Libraries

```python
import math
import numpy as np
```

```python
import matplotlib.pyplot as plt
plt.style.use('ggplot')
#print(plt.style.available)
```

### Utils functions

```latex
%%latex
Entropy formula
\begin{align}
  H(X) = -\sum_{x}{p(x) * log_2\,{p(x)}}
\end{align}
```

Entropy formula
\begin{align}
  H(X) = -\sum_{x}{p(x) * log_2\,{p(x)}}
\end{align}

```python
def entropy(p_x):
    h_sum = float()
    for item in p_x:
        h_sum += item * math.log(item,2)
    return -h_sum
```

### Entropy vs probability

In the example below, we compare probability of some random variable X and the entropy of that variable.

```python
%%time

x_r = np.arange(0.001, 0.999, 0.01)
print("Lenght of x samples: %i" % len(x_r))
# starting from 0.001 because log_2(0) not defined
# eg: math.log(0,2) will yield "ValueError: math domain error"

#y_r = []
#for x in x_r:
#    args = [x, 1.0-x]
#    y_r.append(entropy(args))
# <=>
y_r = list(map(lambda x: entropy([x, 1.0-x]), x_r))

plt.plot(x_r, y_r, c='b')
plt.xlabel('Probability - p(x)') #Pr(X=1) ?
plt.ylabel('Entropy - H(x) - bits') #p(x)

plot_margin = 0.01
x0, x1, y0, y1 = plt.axis()
plt.axis((x0 - plot_margin,
          x1 + plot_margin,
          y0 - plot_margin,
          y1 + plot_margin))
plt.show()
```

What can be seen in this graph is that we get the most **bits of information** when our sets are relatively equal divided (equal probability of encountering either of the classes).

**`Why is that ?`**
Because **when we have the biggest entropy** (approaching 1), we have the **most chaos/impurity in our data**, then we **gain the most information in a random experiment** when we have the **least assurance of predicting which class** will be randomly selected next.
(alternatively, that assurance will be much higher if we know there is only **`10% chance`** of encountering one class versus **`90% chance`** of another).

### An example, based on [this course notes](https://homes.cs.washington.edu/~shapiro/EE596/notes/InfoGain.pdf)

```python
%%time

N = 30
N1 = 16
N2 = 14

x1 = 0.9*np.random.rand(N1)
y1 = 0.9*np.random.rand(N1)
#print(np.shape(x1))

x2 = 0.9*np.random.rand(N2)
y2 = 0.9*np.random.rand(N2)
#print(np.shape(x2))

plt.scatter(x1, y1, s=90, marker='o', c='g')
plt.scatter(x2, y2, s=90, marker='^', c='m')

plt.show()
```

```python
# The only thing valuable of the above are how many members each class has
# in this case, fairly equal

#p_x = [16/30, 14/30]
p_x = [N1/N, N2/N]
print("Entropy result: %s" % entropy(p_x))
```

    Entropy result: 0.9967916319816366

```python

```


```python
import sympy as sm
x = sm.symbols('x')
```

# Derivatives
> ## $ \frac{d}{dx}(\frac{1+sinn\,x}{1-cos\,x})^2 $

```python
sm.diff((1+sm.sin(x)) / (1 - sm.cos(x))**2,x)
```

$\displaystyle \frac{\cos{\left(x \right)}}{\left(1 - \cos{\left(x \right)}\right)^{2}} - \frac{2 \left(\sin{\left(x \right)} + 1\right) \sin{\left(x \right)}}{\left(1 - \cos{\left(x \right)}\right)^{3}}$

>  ## $
\frac{d}{dx}(log_5(x))^{x/2}
$

```python
sm.diff(sm.log(x,5)**(x/2),x)
```

$\displaystyle \left(\frac{\log{\left(x \right)}}{\log{\left(5 \right)}}\right)^{\frac{x}{2}} \left(\frac{\log{\left(\frac{\log{\left(x \right)}}{\log{\left(5 \right)}} \right)}}{2} + \frac{1}{2 \log{\left(x \right)}}\right)$

> ## $
\frac{d}{dx}f(x+g(x)) \\
c.f \quad \xi ,\, \Xi \to xi
$

```python
f,g = sm.symbols('f g',cls=sm.Function)
g = g(x)
f = f(x + g)
sm.diff(f,x)
```

$\displaystyle \left(\frac{d}{d x} g{\left(x \right)} + 1\right) \left. \frac{d}{d \xi_{1}} f{\left(\xi_{1} \right)} \right|_{\substack{ \xi_{1}=x + g{\left(x \right)} }}$

# Basic AntiDerivative
> ## $ \int csc(x)cot(x)dx$

```python
sm.integrate(sm.csc(x)*sm.cot(x),x)
```

$\displaystyle - \frac{1}{\sin{\left(x \right)}}$

> ## $
\int 4sec(3x)tan(3x)dx
$

```python
sm.integrate(4*sm.sec(3*x)*sm.tan(3*x),x)
```

$\displaystyle \frac{4}{3 \cos{\left(3 x \right)}}$

```python

```


# Skew G-Jensen-Shannon divergence

```python
import matplotlib.pyplot as plt
import numpy as np

from matplotlib import cm

from utils import PlotParams
```

```python
plotter = PlotParams()
plotter.set_params()
FIG_DIR = os.path.join(os.pardir, 'figs')
```

```python
def gaussian(x, mu, sig):
    return 1. / (np.sqrt(2. * np.pi) * sig) * np.exp(-np.power(x - mu, 2.) / (2 * np.power(sig, 2.)))
```

## One-dimensional normal distribution comparison

### Distributions

```python
x = np.linspace(-5, 5, 500)

example_num = 1
m_1, s_1 = 0, 1
m_2, s_2 = 2, 0.5

p = gaussian(x, m_1, s_1)
q = gaussian(x, m_2, s_2)

plt.plot(x, p, 'b', label='$p(z)\sim\mathcal{N}$'+f'({m_1},{s_1})')
plt.plot(x, q, 'r', label='$q(z|x)\sim\mathcal{N}$'+f'({m_2},{s_2})')
plt.xlim(-5, 5), plt.ylim(bottom=0)
plt.xlabel('$z$')
plt.legend()
plt.savefig(os.path.join(FIG_DIR, f'gaussians-1D_{example_num}.pdf'), bbox_inches='tight')
```

### Kullback-Leibler divergence
* Clearly asymmetric

```python
kl_qp = q * np.log(q / p)
plt.fill_between(x, kl_qp, np.zeros_like(kl_qp), color='red', alpha=0.5, label=r'KL$(q\parallel p)$')
kl_pq = p * np.log(p / q)
plt.fill_between(x, kl_pq, np.zeros_like(kl_pq), color='blue', alpha=0.5, label=r'KL$(p\parallel q)$')
plt.xlim(-5, 5)
plt.xlabel(r'$x$')
_ = plt.legend(loc='upper left')
```

### Jensen-Shannon divergence
Jenson-Shannon divergence is the average divergence to the **arithmetic** mean
\begin{align}
    \textrm{JS}(q\parallel p) = \frac{1}{2}\textrm{KL}\left(p\parallel \frac{p+q}{2}\right) + \frac{1}{2}\textrm{KL}\left(q\parallel\frac{p+q}{2}\right).
\end{align}

```python
_ = range_ = np.linspace(0, 1, 11)
cmap = cm.jet(range_)
for a, c in zip(range_, cmap):
    m = a * p + (1 - a) * q
    plt.plot(x, m, c=c, label=r'$\alpha=$'+f'{a:.1f}')
plt.xlim(-5, 5), plt.ylim(bottom=0)
_ = plt.xlabel('$z$')
```

```python
mo = (p + q) / 2
```

```python
plt.plot(x, p, 'b')
plt.plot(x, q, 'r')
plt.plot(x, mo, 'g:', label=r'$m(x)=\frac{1}{2}(p(z)+q(z|x))$')
plt.xlim(-5, 5), plt.ylim(bottom=0)
plt.xlabel('$z$')
_ = plt.legend()
plt.savefig(os.path.join(FIG_DIR, f'gaussians-1D_JS-0.5.pdf'), bbox_inches='tight')
```

KL divergence terms to **arithmetic** mean in JS divergence

```python
kl_qm = q * np.log(q / mo)
plt.fill_between(x, kl_qm, np.zeros_like(kl_qm), color='red', alpha=0.5, label=r'KL$(q\parallel m)$')
kl_pm = p * np.log(p / mo)
plt.fill_between(x, kl_pm, np.zeros_like(kl_pm), color='blue', alpha=0.5, label=r'KL$(p\parallel m)$')
plt.xlim(-5, 5)
plt.xlabel('$z$')
_ = plt.legend(loc='upper left')
```

Final (point-wise) JS divergence

```python
js_qp = (kl_pm + kl_qm) / 2
plt.fill_between(x, js_qp, np.zeros_like(js_qp), color='black', alpha=0.5, label=r'JS$(p\parallel q)$')
plt.xlim(-5, 5), plt.ylim(bottom=0)
plt.xlabel('$z$')
_ = plt.legend(loc='upper left')
```

## Skew geometric-Jensen-Shannon

In one dimension, the matrix harmonoic barycentre for $\Sigma_{\alpha}$
\begin{align}
    \Sigma_{\alpha} &= \left((1-\alpha)\Sigma_{1}^{-1} + \alpha\Sigma_{2}^{-1}\right)^{-1} \\
    \mu_{\alpha} &= \Sigma_{\alpha}\left((1-\alpha)\Sigma_{1}^{-1}\mu_{1}+\alpha\Sigma_{2}^{-1}\mu_{2}\right),
\end{align}
reduces to a weighted harmonic mean:
\begin{align}
        \sigma_{\alpha}^{2} &= \frac{1}{(1-\alpha)\frac{1}{\sigma_{1}^{2}}+\alpha\frac{1}{\sigma_{2}^{2}}} \\
        \mu_{\alpha}^{2} &= \sigma_{\alpha}^{2} \left((1-\alpha)\frac{\mu_{1}}{\sigma_{1}^{2}}+\alpha\frac{\mu_{2}}{\sigma_{2}^{2}}\right)
\end{align}

```python
def get_s_a(m_1, s_1, m_2, s_2, a=0.5):
    s_a = 1 / ((1 - a) / s_1 + a / s_2)
    m_a = s_a * ((1 - a) * m_1 / s_1 + a * m_2 / s_2)

    return m_a, s_a
```

```python
m_a, s_a = get_s_a(m_1, s_1, m_2, s_2, a=0.5)
m = gaussian(x, m_a, s_a)
```

Skew geometric-Jenson-Shannon divergence is the weighted average divergence to the **geometric** mean
\begin{align}
    \textrm{JS}(q\parallel p) = (1-\alpha)\textrm{KL}\left(p, G_{\alpha}(p,q)\right) + \alpha\textrm{KL}\left(q, G_{\alpha}(p,q)\right).
\end{align}

```python
plt.plot(x, p, 'b')
plt.plot(x, q, 'r')
# plt.plot(x, mo, 'g:', label=r'$\frac{1}{2}(p+q)$')
plt.plot(x, m, 'g', label=r'$\sqrt{qp}$')
plt.xlim(-5, 5), plt.ylim(bottom=0)
plt.xlabel('$z$')
_ = plt.legend()
plt.savefig(os.path.join(FIG_DIR, f'gaussians-1D_GJS-0.5.pdf'), bbox_inches='tight')
```

$\alpha=[0.0,\ldots,1.0]$

```python
_ = range_ = np.linspace(0, 1, 11)
cmap = cm.jet(range_)
for a, c in zip(range_, cmap):
    m_a, s_a = get_s_a(m_1, s_1, m_2, s_2, a=a)
    m = gaussian(x, m_a, s_a)
    plt.plot(x, m, c=c, label=r'$\alpha=$'+f'{a:.1f}')
plt.xlim(-5, 5), plt.ylim(bottom=0)
plt.xlabel('$z$')
plt.legend()
```

```python
m_a, s_a = get_s_a(m_1, s_1, m_2, s_2, a=0.5)
m = gaussian(x, m_a, s_a)
```

KL divergence terms to **geometric** mean in G-JS divergence

```python
kl_qm = q * np.log(q / m)
plt.fill_between(x, kl_qm, np.zeros_like(kl_qm), color='red', alpha=0.5, label=r'KL$(q\parallel m)$')
kl_pm = p * np.log(p / m)
plt.fill_between(x, kl_pm, np.zeros_like(kl_pm), color='blue', alpha=0.5, label=r'KL$(p\parallel m)$')
plt.xlim(-5, 5)
plt.xlabel('$z$')
_ = plt.legend(loc='upper left')
```

Final (point-wise) G-JS divergence

```python
gjs_qp = (kl_pm + kl_qm) / 2
plt.fill_between(x, gjs_qp, np.zeros_like(gjs_qp), color='black', alpha=0.5, label=r'G$^{\alpha}$-JS$(q\parallel p)$')
plt.xlim(-5, 5)
plt.xlabel('$z$')
_ = plt.legend(loc='upper left')
```

```python
plt.plot(x, p, 'b')
plt.plot(x, q, 'r')
plt.plot(x, js_qp, 'k', label=r'JS$(q\parallel p)$')
plt.plot(x, gjs_qp, 'k:', label=r'G$^{\alpha}$-JS$(q\parallel p)$')
plt.xlim(-5, 5)
plt.xlabel('$z$')
plt.legend()
plt.savefig(os.path.join(FIG_DIR, f'gaussians-divergences-1D_{example_num}.pdf'), bbox_inches='tight')
```

```python

```

```python

```


# Sine Series Problem: Error Analysis (Student notebook)

* *Computational Physics*: Ch 2.5
* In-class Activity

## Problem: Summing Series: sin(x)

Evaluate the $\sin$ function from its series representation
$$
\sin x = x - \frac{x^3}{3!} + \frac{x^5}{5!} - \frac{x^7}{7!} + \dots
$$

### Explicit sum
A naive algorithm is to sum the series up to the $N$th term:
$$
\sin x \approx \sum_{n=1}^N \frac{(-1)^{n-1} x^{2n-1}}{(2n - 1)!}
$$

Problems:

- How to decide when to stop summing?
- Division of large terms (overflows!)
- Powers and factorials are very expensive to compute.

### Recursive sum
Better approach: Build up series terms $a_n$ using previous term $a_{n-1}$ through a recursion:

\begin{align}
a_n &= a_{n-1} \times q_n\\
a_n &= \frac{(-1)^{n-1} x^{2n-1}}{(2n - 1)!} = \frac{(-1)^{n-2} x^{2n-3}}{(2n - 3)!} \frac{-x^2}{(2n - 1)(2n - 2)}\\
a_n & = a_{n-1} \frac{-x^2}{(2n - 1)(2n - 2)}
\end{align}

Accuracy of this approach? Not clear in absolute terms but we can make the assumption that the error is approximately the last term in the sum, $a_N$. Hence we can strive to make the relative error smaller than the machine precision
$$
\left| \frac{a_N}{\sum_{n=1}^N a_n} \right| < \epsilon_m
$$

## Task

In file `series.py` implement a function `sin_recursive(x, N=100)` that
* computes $\sin(x)$ using the **Recursive sum** approach using $N$ iterations
* returns the value of $\sin(x)$

## Error analysis

```python
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
```

Import your `series.py` as a module:

```python
import series
```

NOTE: If you changed code in `series` you will need to **reload the module**: execute the following cell whenever you changed `series.py`.

```python
from importlib import reload
reload(series)
```

    <module 'series' from '/Volumes/ASU/oliver/Documents/Teaching/ASU/CompPhys_PHY494/2021/PHY494-resources/08_errors/erroranalysis/series.py'>

Test the implementation against the "exact" numpy function `np.sin()` and show the relative error as a function of $1 \le N < 1000$.

Report and plot
1. Try out `x =` $\pi/3, 2\pi, 7.89\pi, -12.3\pi, 14.78\pi$
2. relative error `abs(sin_series(x) - sin(x))/abs(sin(x))` as function of $N$

(Use a log-log plot.)

#### Testing code

Implementation of the error calculation against the numpy reference implementation `np.sin`:

```python
def relerror_sin(N_values, x):
    # TODO
    pass
```

#### Analyze recursive implementation

Plot for $x = \pi/3$ and for $x = -7.89\pi$

```python

```

Plot all x values:

```python

```

```python

```


# Machine Learning Calculus

```python
from sympy import *
import numpy as np
import math

x = Symbol('x')
y = Symbol('y')
```

```python
limit(1/x**2,x,0)
```

    oo

```python
# Find the limit of Functions
from sympy import Limit, Symbol, S

Limit(1/x, x, S.Infinity)
```

    Limit(1/x, x, oo, dir='-')

```python
# To find the value
l = Limit(1/x, x, S.Infinity)
l.doit()
```

    0

```python
# Differentiation
diff(15*x**100-3*x**12+5*x-46)
```

    1500*x**99 - 36*x**11 + 5

```python
# Constant
diff(99)
```

    0

```python
# Multiplication by Constant
diff(3*x)
```

    3

```python
# Power Rule
diff(x**3)
```

    3*x**2

```python
# Sum Rule
diff(x**2+3*x)
```

    2*x + 3

```python
# Difference Rule
diff(x**2-3*x)
```

    2*x - 3

```python
# Product Rule
diff(x**2*x)
```

    3*x**2

```python
# Chain Rule
diff(ln(x**2))
```

    2/x

```python
# Example
diff(9*(x+x**2))
```

    18*x + 9

```python
# Matrix Calculus
a = diff(3*x**2*y,x)
b = diff(3*x**2*y,y)
c = diff(2*x+8*y**7,x)
d = diff(2*x+y**8,y)
```

```python
print(a)
print(b)
print(c)
print(d)
```

    6*x*y
    3*x**2
    2
    8*y**7

```python
import numpy as np
Matrix_Calculus = np.matrix([[a, c], [b,d]])
Matrix_Calculus
```

    matrix([[6*x*y, 2],
            [3*x**2, 8*y**7]], dtype=object)

```python
# Chain Rule
diff(ln(sin(x**3)**2))
```

    6*x**2*cos(x**3)/sin(x**3)

```python
# Sigmoid
# d/dx S(x)=S(x)(1−S(x))
diff(1/(1+math.e**-x),x)
```

    1.0*2.71828182845905**(-x)/(1 + 2.71828182845905**(-x))**2

```python
# Integral
import sympy
sympy.init_printing()
sympy.integrate(2*x, (x, 1, 0))
```

```python
# Jacobian
from sympy import sin, cos, Matrix
from sympy.abc import rho, phi

X = Matrix([rho*cos(phi), rho*sin(phi), rho**2])
Y = Matrix([rho, phi])

X.jacobian(Y)
```

$$\left[\begin{matrix}\cos{\left (\phi \right )} & - \rho \sin{\left (\phi \right )}\\\sin{\left (\phi \right )} & \rho \cos{\left (\phi \right )}\\2 \rho & 0\end{matrix}\right]$$

```python
# Example Calculating the Jacobian
# Equation
eq = x**2*y + 3/4*x*y + 10
eq
```

```python
# Jacobian matrix
x, y, z = symbols('x y z')
Matrix([sin(x) + y, cos(y) + x, z]).jacobian([x, y, z])
```

$$\left[\begin{matrix}\cos{\left (x \right )} & 1 & 0\\1 & - \sin{\left (y \right )} & 0\\0 & 0 & 1\end{matrix}\right]$$

```python
# (x,y)=(0,0)
x1 = -y
x2 = x - 2*y * (2-x**2)
J = sympy.Matrix([x1,x2])
J.jacobian([x,y])
```

$$\left[\begin{matrix}0 & -1\\4 x y + 1 & 2 x^{2} - 4\end{matrix}\right]$$

```python
J.jacobian([x,y]).subs([(x,0), (y,0)])
```

$$\left[\begin{matrix}0 & -1\\1 & -4\end{matrix}\right]$$

```python
# Derivatives of x
de_x = diff(x**2*y + 3/4*x*y + 10, x)
de_x
```

```python
# Derivatives of y
de_y = diff(x**2*y + 3/4*x*y + 10, y)
de_y
```

```python
# Example
F = sympy.Matrix([de_x,de_y])
F.jacobian([x,y])
```

$$\left[\begin{matrix}2 y & 2 x + 0.75\\2 x + 0.75 & 0\end{matrix}\right]$$

```python
F.jacobian([x,y]).subs([(x,0), (y,0)])
```

$$\left[\begin{matrix}0 & 0.75\\0.75 & 0\end{matrix}\right]$$

```python
# Hessian
from sympy import Function, hessian, pprint
from sympy.abc import x, y
f = Function('f')(x, y)
g1 = Function('g')(x, y)
g2 = x**2+y**2
pprint(hessian(f, (x,y), [g1, g2]))
```

    ⎡                   ∂               ∂            ⎤
    ⎢     0        0    ──(g(x, y))     ──(g(x, y))  ⎥
    ⎢                   ∂x              ∂y           ⎥
    ⎢                                                ⎥
    ⎢     0        0        2⋅x             2⋅y      ⎥
    ⎢                                                ⎥
    ⎢                     2               2          ⎥
    ⎢∂                   ∂               ∂           ⎥
    ⎢──(g(x, y))  2⋅x   ───(f(x, y))   ─────(f(x, y))⎥
    ⎢∂x                   2            ∂y ∂x         ⎥
    ⎢                   ∂x                           ⎥
    ⎢                                                ⎥
    ⎢                     2               2          ⎥
    ⎢∂                   ∂               ∂           ⎥
    ⎢──(g(x, y))  2⋅y  ─────(f(x, y))   ───(f(x, y)) ⎥
    ⎢∂y                ∂y ∂x              2          ⎥
    ⎣                                   ∂y           ⎦

```python
import numpy as np
import matplotlib.pyplot as plt

def tanh(x):
    return np.tanh(x)

X = np.linspace(-5, 5, 100)
plt.plot(X, tanh(X),'b')
plt.xlabel('X Axis')
plt.ylabel('Y Axis')
plt.title('Neural Networks - Activation Function')
plt.grid()
plt.text(4, 0.8, r'$\sigma(x)=\tanh{(x)}}$', fontsize=16)
plt.show()
```

    <Figure size 640x480 with 1 Axes>

```python
def sigma(x):
    return (np.exp(x) - np.exp(-x)) / (np.exp(x) + np.exp(-x))

X = np.linspace(-5, 5, 100)
plt.plot(X, sigma(X),'b')
plt.xlabel('X Axis')
plt.ylabel('Y Axis')
plt.title('Sigmoid Function')
plt.grid()
plt.text(4, 0.8, r'$\sigma(x)=\frac{e^{x}-e^{-x}}{e^{x}+e^{-x}}$', fontsize=16)
plt.show()
```

```python
def derivatives_sigma(x):
    return 1 / (np.cosh(x))**2

X = np.linspace(-5, 5, 100)
plt.plot(X, derivatives_sigma(X),'b')
plt.xlabel('X Axis')
plt.ylabel('Y Axis')
plt.title('Sigmoid Function')
plt.grid()
plt.text(4, 0.8, r'$\sigma(x)=\frac{1}{cosh^2(x)}$', fontsize=16)
plt.show()
```

```python
def derivatives_sigma(x):
    return 4 / (np.exp(x)+np.exp(-x))**2

X = np.linspace(-5, 5, 100)
plt.plot(X, derivatives_sigma(X),'b')
plt.xlabel('X Axis')
plt.ylabel('Y Axis')
plt.title('Sigmoid Function')
plt.grid()
plt.text(4, 0.8, r'$\sigma(x)=\frac{4}{(e^{x}+e^{-x})^2}$', fontsize=16)
plt.show()
```

```python
# Newton_Raphson
# f(x) - the function of the polynomial
from sympy import *

def f(x):
    function = x**3 - x - 1
    return function

def derivative(x): #function to find the derivative of the polynomial
    derivative = diff(f(x), x)
    return derivative

def Newton_Raphson(x):
	return (x - (f(x) / derivative(x)))

Newton_Raphson(x)
```

```python
# Advanced Chain Rule
# F(y)=ln(1−5y2+y3)
from sympy import *

y = symbols('y')
F = symbols('F', cls=Function)

Derivative(ln(1 - 5*y**2 + y**3)).doit()
```

```python
diff(ln(1 - 5*y**2 + y**3))
```


# Vector

## Create a 3D Cartesian coordinate system

```python
# Create a 3D Cartesian coordinate
from sympy.vector import CoordSys3D

N = CoordSys3D('')
```

## Add unit vector along the x axis $$N\vec{i}$$

```python
# Add unit vector along the axes
N.i
```

$\displaystyle \mathbf{\hat{i}_{}}$

```python
type(N.i)
```

    sympy.vector.vector.BaseVector

## Create a vector A $$\vec{A} = N\vec{i} + N\vec{j} + N\vec{k}$$

```python
# Create a vector A
vect_A = N.i + N.j + N.k
vect_A
```

$\displaystyle \mathbf{\hat{i}_{}} + \mathbf{\hat{j}_{}} + \mathbf{\hat{k}_{}}$

## Add an operation to the vector $$3\times N\vec{j}$$

```python
# Add an operation to the vector
vect_A = N.i + 3*N.j + N.k
vect_A
```

$\displaystyle \mathbf{\hat{i}_{}} + (3)\mathbf{\hat{j}_{}} + \mathbf{\hat{k}_{}}$

## Create a vector B $$\vec{B} = 2\times N\vec{i} + N\vec{j} + N\vec{k}$$

```python
# Create a vector B
vect_B = 2*N.i + N.j + N.k
vect_B
```

$\displaystyle (2)\mathbf{\hat{i}_{}} + \mathbf{\hat{j}_{}} + \mathbf{\hat{k}_{}}$

## Vector Dot Product $$\vec{A}\cdot\vec{B}$$

```python
# vector A . vector B
vect_A & vect_B
```

$\displaystyle 6$

## Vector Cross Product $$\vec{A}\times\vec{B}$$

```python
# vector A x vector B
vect_A ^ vect_B
```

$\displaystyle (2)\mathbf{\hat{i}_{}} + \mathbf{\hat{j}_{}} + (-5)\mathbf{\hat{k}_{}}$


# Getting started with TensorFlow (Eager Mode)

**Learning Objectives**
  - Understand difference between Tensorflow's two modes: Eager Execution and Graph Execution
  - Practice defining and performing basic operations on constant Tensors
  - Use Tensorflow's automatic differentiation capability

## Introduction
**Eager Execution**

Eager mode evaluates operations immediatley and return concrete values immediately. To enable eager mode simply place `tf.enable_eager_execution()` at the top of your code. We recommend using eager execution when prototyping as it is intuitive, easier to debug, and requires less boilerplate code.

**Graph Execution**

Graph mode is TensorFlow's default execution mode (although it will change to eager with TF 2.0). In graph mode operations only produce a symbolic graph which doesn't get executed until run within the context of a tf.Session(). This style of coding is less inutitive and has more boilerplate, however it can lead to performance optimizations and is particularly suited for distributing training across multiple devices. We recommend using delayed execution for performance sensitive production code.

```python
# Ensure that we have Tensorflow 1.13.1 installed.
!pip3 freeze | grep tensorflow==1.13.1 || pip3 install tensorflow==1.13.1
```

    tensorflow==1.13.1

```python
import tensorflow as tf
print(tf.__version__)
```

    1.13.1

## Eager Execution

```python
tf.enable_eager_execution()
```

### Adding Two Tensors

The value of the tensor, as well as its shape and data type are printed

```python
a = tf.constant(value = [5, 3, 8], dtype = tf.int32)
b = tf.constant(value = [3, -1, 2], dtype = tf.int32)
c = tf.add(x = a, y = b)
print(c)
```

    tf.Tensor([ 8  2 10], shape=(3,), dtype=int32)

#### Overloaded Operators
We can also perform a `tf.add()` using the `+` operator. The `/,-,*` and `**` operators are similarly overloaded with the appropriate tensorflow operation.

```python
c = a + b # this is equivalent to tf.add(a,b)
print(c)
```

    tf.Tensor([ 8  2 10], shape=(3,), dtype=int32)

### NumPy Interoperability

In addition to native TF tensors, tensorflow operations can take native python types and NumPy arrays as operands.

```python
import numpy as np

a_py = [1,2] # native python list
b_py = [3,4] # native python list

a_np = np.array(object = [1,2]) # numpy array
b_np = np.array(object = [3,4]) # numpy array

a_tf = tf.constant(value = [1,2], dtype = tf.int32) # native TF tensor
b_tf = tf.constant(value = [3,4], dtype = tf.int32) # native TF tensor

for result in [tf.add(x = a_py, y = b_py), tf.add(x = a_np, y = b_np), tf.add(x = a_tf, y = b_tf)]:
    print("Type: {}, Value: {}".format(type(result), result))
```

    Type: <class 'tensorflow.python.framework.ops.EagerTensor'>, Value: [4 6]
    Type: <class 'tensorflow.python.framework.ops.EagerTensor'>, Value: [4 6]
    Type: <class 'tensorflow.python.framework.ops.EagerTensor'>, Value: [4 6]

You can convert a native TF tensor to a NumPy array using .numpy()

```python
a_tf.numpy()
```

    array([1, 2], dtype=int32)

### Linear Regression

Now let's use low level tensorflow operations to implement linear regression.

Later in the course you'll see abstracted ways to do this using high level TensorFlow.

#### Toy Dataset

We'll model the following function:

\begin{equation}
y= 2x + 10
\end{equation}

```python
X = tf.constant(value = [1,2,3,4,5,6,7,8,9,10], dtype = tf.float32)
Y = 2 * X + 10
print("X:{}".format(X))
print("Y:{}".format(Y))
```

    X:[ 1.  2.  3.  4.  5.  6.  7.  8.  9. 10.]
    Y:[12. 14. 16. 18. 20. 22. 24. 26. 28. 30.]

#### Loss Function

Using mean squared error, our loss function is:
\begin{equation}
MSE = \frac{1}{m}\sum_{i=1}^{m}(\hat{Y}_i-Y_i)^2
\end{equation}

$\hat{Y}$ represents the vector containing our model's predictions:
\begin{equation}
\hat{Y} = w_oX + w_1
\end{equation}

#### **Exercise 1**

The function `loss_mse` below takes four arguments: the tensors $X$, $Y$ and the weights $w_0$ and $w_1$. Complete the function below to compute the Mean Square Error (MSE). Hint: [check out](https://www.tensorflow.org/api_docs/python/tf/math/reduce_mean) the `tf.reduce_mean` function.

```python
def loss_mse(X, Y, w0, w1):
    yhat = w0 * X + w1
    return tf.math.reduce_mean((yhat - Y)**2)
```

#### Gradient Function

To use gradient descent we need to take the partial derivative of the loss function with respect to each of the weights. We could manually compute the derivatives, but with Tensorflow's automatic differentiation capabilities we don't have to!

During gradient descent we think of the loss as a function of the parameters $w_0$ and $w_1$. Thus, we want to compute the partial derivative with respect to these variables. The `params=[2,3]` argument tells TensorFlow to only compute derivatives with respect to the 2nd and 3rd arguments to the loss function (counting from 0, so really the 3rd and 4th).

```python
# Counting from 0, the 2nd and 3rd parameter to the loss function are our weights
grad_f = tf.contrib.eager.gradients_function(f = loss_mse, params = [2, 3])
```

    WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.
    For more information, please see:
      * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md
      * https://github.com/tensorflow/addons
    If you depend on functionality not listed there, please file an issue.

#### Training Loop

Here we have a very simple training loop that converges. Note we are ignoring best practices like batching, creating a separate test set, and random weight initialization for the sake of simplicity.

#### **Exercise 2**

Complete the code to update the parameters $w_0$ and $w_1$ according to the gradients `d_w0` and `d_w1` and the specified learning rate.

```python
%%time
STEPS = 1000
LEARNING_RATE = .02

# Initialize weights
w0 = tf.constant(value = 0.0, dtype = tf.float32)
w1 = tf.constant(value = 0.0, dtype = tf.float32)

for step in range(STEPS):
    #1. Calculate gradients
    d_w0, d_w1 = grad_f(X, Y, w0, w1) # derivatives calculated by tensorflow!

    #2. Update weights
    w0 = w0 - d_w0 * LEARNING_RATE
    w1 = w1 - d_w1 * LEARNING_RATE

    #3. Periodically print MSE
    if step % 100 == 0:
        print("STEP: {} MSE: {}".format(step,loss_mse(X, Y, w0, w1)))

# Print final MSE and weights
print("STEP: {} MSE: {}".format(STEPS,loss_mse(X, Y, w0, w1)))
print("w0:{}".format(round(float(w0), 4)))
print("w1:{}".format(round(float(w1), 4)))
```

    STEP: 0 MSE: 167.6111297607422
    STEP: 100 MSE: 3.5321757793426514
    STEP: 200 MSE: 0.6537718176841736
    STEP: 300 MSE: 0.12100745737552643
    STEP: 400 MSE: 0.022397063672542572
    STEP: 500 MSE: 0.004145540297031403
    STEP: 600 MSE: 0.0007674093940295279
    STEP: 700 MSE: 0.00014202017337083817
    STEP: 800 MSE: 2.628635775181465e-05
    STEP: 900 MSE: 4.86889211970265e-06
    STEP: 1000 MSE: 9.178326081382693e-07
    w0:2.0003
    w1:9.9979
    CPU times: user 1.02 s, sys: 12 ms, total: 1.03 s
    Wall time: 1.02 s

## Bonus

Try modelling a non-linear function such as: $y=xe^{-x^2}$

Hint: Creating more training data will help. Also, you will need to build non-linear features.

```python
from matplotlib import pyplot as plt
%matplotlib inline

X = tf.constant(value = np.linspace(0,2,1000), dtype = tf.float32)
Y = X*np.exp(-X**2) * X
plt.plot(X, Y)
```

```python
len(X)
```

    1000

```python
# start by doing a polynomial approximation. Need to decrease learning rate by a lot to go above fourth-order
def make_features(X):
    features = [X]
    features.append(tf.ones_like(X))  # Bias.
    features.append(X**2)
    features.append(X**3)
    features.append(X**4)
    features.append(X**5)
    features.append(X**6)
    return tf.stack(features, axis=1)

def make_weights(n_weights):
    W = [tf.constant(value = 0.0, dtype = tf.float32) for _ in range(n_weights)]
    return tf.expand_dims(tf.stack(W),-1)

def predict(X, W):
    Y_hat = tf.matmul(X, W)
    return tf.squeeze(Y_hat, axis=-1)

def loss_mse(X, Y, W):
    Y_hat = predict(X, W)
    return tf.reduce_mean(input_tensor = (Y_hat - Y)**2)

X = tf.constant(value = np.linspace(0,2,1000), dtype = tf.float32)
Y = np.exp(-X**2) * X

grad_f = tf.contrib.eager.gradients_function(f = loss_mse, params=[2])
```

```python
STEPS = 20000
LEARNING_RATE = .0001

# Weights/features.
Xf = make_features(X)
W = make_weights(Xf.get_shape()[1].value)

# For plotting
steps = []
losses = []

plt.figure()
for step in range(STEPS):
    #1. Calculate gradients
    dW = grad_f(Xf, Y, W)[0]
    #2. Update weights
    W -= dW * LEARNING_RATE
    #3. Periodically print MSE
    if step % 10 == 0:
        loss = loss_mse(Xf, Y, W)
        steps.append(step)
        losses.append(loss)
        plt.clf()
        plt.plot(steps, losses)
# Print final MSE and weights
print("STEP: {} MSE: {}".format(STEPS,loss_mse(Xf, Y, W)))

# Plot results
plt.figure()
plt.plot(X, Y, label='actual')
plt.plot(X, predict(Xf, W), label='predicted')
plt.legend()
```

Copyright 2019 Google Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License

```python
x = tf.constant([[3,5,7], [4,6,8]])
```

```python

```


This notebook demonstrates using SciPy to solve ODEs and Matplotlib for plotting

# Geodesics in Schwarzschild spacetime

The Schwarzschild spacetime describes an isolated spherically symmetric black hole. The (timelike) geodesic equations give the motion of a test mass around the black hole. In this notebook we give the equations, place them into first-order form, numerically integrate them and then plot the resulting orbit.

In this notebook we use geometrized units such that $G = c = 1$. The Schwarzschild black hole has mass $M$, and the Schwarzschild coordinates are $(t,r,\theta, \phi)$.

## Equations of motion

Let the coordinates of the test body be denoted by $x^\mu_p(\tau) = \{t_p(\tau), r_p(\tau), \theta_p(\tau),\phi_p(\tau)\}$ where $\tau$ is the propertime (propertime is the time as mearsured by a clock movine along with the test body$. Using the Euler-Lagrange equations one can derive the equations of motion a test body in orbit about the black hole. These are given by the ODEs:

$$\begin{align}
\frac{dt}{d\tau} &= E^2(1-2M/r)^{-1}\\
\left(\frac{dr}{d\tau}\right)^2 &= E^2 - \left(1- \frac{2M}{r}\right)\left(1 + \frac{L^2}{r^2}\right)\\
\frac{d\phi}{d\tau} & = \frac{L}{r^2}
\end{align}$$

where $E$ and $L$ are the energy and angular momentum as measured at infinity.

Note due to spherical symmetry, without loss of generality we can set $\theta=\pi/2$ and $\theta$ does not evolve over the orbit.

## Manipulating the radial equation

The radial equation above is written in terms of first-order derivaties but it is not especially useful for numerical integration. That is because to use it you have to take a square root, which brings with it a $\pm$. The plus represents the outgoing motion from periastron to apastron, and the minus the ingoing motion. Switching between these two is difficult numerically.

A nice solution is to take a derviative of the whole radial equation w.r.t. $\tau$. This results in a second-order differenential equation which removes the troublesome $r'(\tau)$ term. The result of this operation is:

$$\frac{d^2r}{d\tau^2} = \frac{L^2(r-3M)}{r^4} - \frac{M}{r^2}$$

In order to numerically integrate this equation we now need to write this in first-order form.
To do this we introduce $r_1 = dr/d\tau$, then the above equation can be written as two coupled first-order equations:

$$\begin{align}
   \frac{dr}{d\tau}   &= r_1 \\
   \frac{dr_1}{d\tau} &= \frac{L^2(r-3M)}{r^4} - \frac{M}{r^2}
\end{align}$$

## The systems of equations and initial conditions

In order to plot the orbital trajectory we only need the $r(\tau)$ an $\phi(\tau)$ solutions. This makes our set of ODEs:

$$\begin{align}
   \frac{dr}{d\tau}   &= r_1 \\
   \frac{dr_1}{d\tau} &= \frac{L^2(r-3M)}{r^4} - \frac{M}{r^2} \\
   \frac{d\phi}{d\tau} & = \frac{L}{r^2}
\end{align}$$

In order to solve these equations we need to specify the initial conditions at $\tau = \tau_0 $. Without loss of generality we will take $\tau_0 = 0$. We can define $r_0 \equiv r_0$ and $\phi(0) \equiv \phi_0$. Again, w.l.o.g we can take $\phi_0 = 0$.

What about $r_1(0) = dr/d\tau(0) \equiv u^r_0$. For this we can use the original version of the radial equation. Thus we have

$$ u^r_0 = \pm\sqrt{E^2 - \left(1- \frac{2M}{r}\right)\left(1 + \frac{L^2}{r^2}\right)}$$

# The energy and angular-momentum

We will now restrict out attention to bound orbits which have a minimum and maximum radius which we denote by $r_\min/r_\max$, respectively. We can further define two geometric quantities, the semi-latus rectum, $p$, and the orbital eccentricity, $e$. This are given by

$$ p \equiv \frac{2r_\max r_\min}{M(r_\max + r_\min)},\qquad e \equiv \frac{r_\max - r_\min}{r_\max + r_\min}$$

Inverting these equations we get

$$ r_\max = \frac{p}{1-e},\qquad r_\min = \frac{p}{1+e}$$

By looking at the effective potential, the energy and angular momentum can be written in terms of $(p,e)$. This gives

$$ E = \sqrt{\frac{(p-2-2e)(p-2+2e)}{p(p-3-e^2)}}, \qquad L = \sqrt{\frac{p^2 M^2}{p-3-e^2}}$$

In order to represent a stable bound orbit we must have $p > 6+2e$ and $0\le e<1$.

# Writing the code

We now want to write a code to compute the orbital trajectory. To do this we need to write a function that given $p$ and $e$:

1) computes $(E,L)$
2) computes $u^r_0$
3) integrates the ODEs
4) plots the result

In order to plot the results we define the cartesian coordinates via:

$$ x(\tau) = r(\tau) \cos[\phi(\tau)], \qquad y(\tau) = r(\tau) \sin[\phi(\tau)]$$

Next we load the libraries we will need and write the function that computes the geodesic and plots its spatial trajectory. Hereafter we will set $M=1$.

```python
from scipy.integrate import solve_ivp
import numpy as np
import matplotlib.pyplot as plt

# For the code below to work you need SciPy > 1.4 for the args argument in solve_ivp to work
# See the release notes at https://scipy.github.io/devdocs/release.1.4.0.html
# If the below command returns a value less than 1.2 please update SciPy
import scipy
scipy.__version__
```

    '1.4.1'

```python
# Define the geodesic equations
def geodesicEqs(tau, y, L):
    r   = y[0]
    r1  = y[1]

    dr  = r1
    dr1 = L**2*(r - 3)/r**4 - 1/r**2

    dphi = L/r**2

    return [dr, dr1, dphi]
```

```python
def plotGeodesic(p, e, tau_max):

    # Compute the energy and angular momentum
    E = np.sqrt((p-2-2*e)*(p-2+2*e)/(p*(p-3-e**2)))
    L = np.sqrt(p**2/(p-3-e**2))

    # The initial conditions
    # Pick an r0 value between rmin and rmax
    r0 = p
    phi0 = 0

    # Calculate the initial value of the radial four-velocity
    ur0 = - np.sqrt(E**2 - (1-2/r0)*(L**2/r0**2 + 1) )

    # solve the geodesics with solve_ivp from SciPy
    sol = solve_ivp(geodesicEqs, [0, tau_max], [r0, ur0, 0], args = [L], t_eval = np.arange(0, tau_max, 1))

    rVals = sol.y[0]
    phiVals = sol.y[2]

    xVals = rVals*np.cos(phiVals)
    yVals = rVals*np.sin(phiVals)

    # Set the figure size
    plt.figure(figsize = (15,15))

    # Ensure the entire trajectory will be visible
    rmax = p/(1-e)

    plt.xlim(-rmax-1, rmax+1)
    plt.ylim(-rmax-1, rmax+1)

    # Draw a black disk for the black hole. The Schwarzschild radius is r_s = 2M.
    circle1 = plt.Circle((0, 0), 2, color='k')
    plt.gcf().gca().add_artist(circle1)

    # Plot the trajectory
    plt.plot(xVals, yVals)

```

```python
plotGeodesic(10,0.7,1000)
```

```python

```


```python
import numpy as np
import pylab
import scipy
import seaborn as sns
%matplotlib inline
%config InlineBackend.figure_format = 'retina'
```

# Importance sampling

We'll look at an example where we can compute the posterior exactly and compare it to an approximation computed via importance sampling.

Suppose we have the following model:
\begin{align}
\mu &\sim N(0, 9) \\
x_{1:n} &\sim N(\mu, 1)
\end{align}

Because this is a conjugate model, we can compute the posterior exactly:
$$p(\mu | x_{1:n}) = N\left(\mu \,\bigg|\, \frac{\sum_i x_i}{(1/9 + n)}, 9/(1+9n) \right).$$

To compute the importance sampling procedure, we first collect samples from some importance distribution $\mu_i \sim q(\mu)$,
and then compute the unnormalized weights:
$$ w_i \propto \frac{p(\mu_i) p(x_{1:n} | \mu_i)}{q(\mu_i)}$$

The normalized weights $w_i$ give us an approximation to the posterior.
Here $q$ is some distribution with the same support as the posterior that we know how to sample from. In general we want $q$ to have longer tails than the posterior.

```python
def generate_data(n=100):
    mu = np.random.normal(0, 9)
    return np.random.normal(mu, 1, n)
```

```python
n=100
data = generate_data(n)
```

```python
c = 1./9 + n
vals = [scipy.stats.distributions.norm.pdf(m, data.sum()/c, np.sqrt(1./c))
        for m in np.arange(12.5,13.5,0.01)]
```

```python
pylab.plot(np.arange(12.5,13.5,0.01), vals)
```

```python
def importance_sample(data, nsamps=100):
    from scipy.stats.distributions import norm
    mus = np.random.normal(0, 5, nsamps)

    wts = np.zeros(nsamps)
    for i, mu in enumerate(mus):
        wts[i] = norm.logpdf(mu, 0, 3) + sum(norm.logpdf(data, mu, 1)) - norm.logpdf(mu, 0, 5)

    # stabilize weights
    wts - wts.max()
    wts = wts - np.log(sum(np.exp(wts)))

    return np.exp(wts), mus
```

```python
wts, mus = importance_sample(data, 100000)
```

```python
inds = np.argsort(mus)
wts_s = wts[inds]
mus_s = mus[inds]
pylab.plot(mus_s, wts_s)
pylab.xlim((12.5,13.5))
pylab.axvline(sum(mus*wts), ls='dotted')
```

Above we've plotted the importance weights and the posterior mean.

```python

```


```python
import sympy as sm
import sympy.physics.mechanics as me
sm.init_printing()
```

In the video I incorrectly typed `q1, q2, q3, q4 = sm.symbols('q1:5')`. It is corrected below:

```python
q1, q2, q3, q4, q5 = sm.symbols('q1:6')
l1, l2, l3, l4 = sm.symbols('l1:5')
```

```python
N, A, B, C = sm.symbols('N, A, B, C', cls=me.ReferenceFrame)
```

```python
A.orient_body_fixed(N, (q1, q2, 0), 'ZXZ')
A.dcm(N)
```

```python
B.orient_axis(A, q3, A.x)
```

In the video I incorrectly typed `C.orient_body_fixed(B, (q3, q4, 0), 'XZX')`. It is corrected below:

```python
C.orient_body_fixed(B, (q4, q5, 0), 'XZX')
```

```python
r_P1_P2 = l1*A.z
r_P1_P2
```

```python
r_P2_P3 = l2*B.z
r_P2_P3
```

```python
r_P3_P4 = l3*C.z - l4*C.y
r_P3_P4
```

```python
r_P1_P4 = r_P1_P2 + r_P2_P3 + r_P3_P4
r_P1_P4
```

```python
r_P1_P4.express(N)
```

```python
r_P1_P4.express(N).simplify()
```

```python
r_P1_P4.express(B)
```

```python
r_P1_P4.free_symbols(B)
```

```python
r_P1_P4.free_symbols(N)
```

```python

```


# Taylor problem 1.50

This problem attacks the "oscillating skateboard" problem described in Example 1.2 of Taylor.  A Newton's 2nd law analysis leads to the differential equation for the angle $\phi$ in radians:

$
\begin{align}
  \ddot\phi = -\frac{g}{R}\sin\phi
  \;.
\end{align}
$

This is a 2nd order, *nonlinear* differential equation.  We note it is the same equation describing the motion of a simple (undamped, not driven) pendulum.

Problem 1.50 has us solving this equation numerically for particular initial conditions and comparing the plots to the approximate solution based on the small angle approximation for $\sin\phi$.  We'll build up code to find this solution and plot it in steps to illustrate how a notebook evolves.  We don't create the polished version at once!

**Your goal for problem 1.51: Modify the relevant part of this notebook to produce the required figure, print it out, and turn it in with your homework.**

```python
%matplotlib inline
```

```python
import numpy as np
from scipy.integrate import odeint

import matplotlib.pyplot as plt
#plt.rcParams.update({'font.size': 18})

```

We'll define the right-hand side (rhs) of the ordinary differential equations (ODE) using the standard form from the Python basics notebook:

$$\begin{align}
   \frac{d}{dt}\left(\begin{array}{c}
                          \phi \\
                          \dot\phi
                      \end{array}\right)
               = \left(\begin{array}{c}
                          \dot\phi \\
                          -g \sin(\phi)
                       \end{array}\right)
\end{align}$$

```python
def ode_rhs_exact(u_vec, t, *params):
    """
    Right-hand side (rhs) of the differential equation, with
    u_vec = [\phi, \dot\phi] and params = [g, R].  Returns the list of
    d(u_vec)/dt, as prescribed by the differential equation.

    """
    phi, phidot = u_vec  # extract phi and phidot from the passed vector
    g, R = params  # extract g and R from the passed parameters
    return [phidot, -g*np.sin(phi)/R]
```

```python
# parameters
g = 9.8  # in mks units
R = 5    # radius in meters

# absolute and relative tolerances for ode solver
abserr = 1.0e-8
relerr = 1.0e-6

# initial conditions for [phi, phidot]
phi0 = np.pi/2.  # convert initial phi to radians
u0_vec = [phi0, 0.]

t_max = 15.  # integration time
t_pts = np.arange(0, t_max, 0.01)  # array of time points, spaced 0.01

# Integrate the differential equation and read off phi, phidot (note T!)
phi, phidot = odeint(ode_rhs_exact, u0_vec, t_pts, args=(g, R),
                     atol=abserr, rtol=relerr).T
```

```python
fig = plt.figure()
ax = fig.add_subplot(1,1,1)
ax.plot(t_pts, 180./np.pi * phi)
fig.tight_layout()  # make the spacing of subplots nicer
```

**Does the plot make sense for $\phi$?  E.g., does it start at the correct angle? Does it have the behavior you expect (e.g., periodic with constant amplitude)?**

Now let's put this into a function:

```python
def solve_for_phi(phi0, phidot0=0, t_min=0., t_max=1., g=9.8, R=5.):
    """
    Solve the differential equation for the skateboard Example 1.2 in Taylor.
    The result for t, \phi(t) and \dot\phi(t) are returned for a grid with
    t_min < t < t_max and a hardwired (for now) spacing of 0.01 seconds.
    The ODE solver is odeint from scipy, with specified tolerances.
    Units are mks and angles are in radians.
    """

    # absolute and relative tolerances for ode solver
    abserr = 1.0e-8
    relerr = 1.0e-6

    # initial conditions for [phi, phidot]
    u0_vec = [phi0, phidot0]

    t_pts = np.arange(t_min, t_max, 0.01)

    # Integrate the differential equation
    phi, phidot = odeint(ode_rhs_exact, u0_vec, t_pts, args=(g, R),
                     atol=abserr, rtol=relerr).T

    return t_pts, phi, phidot
```

Check that it works (gives the previous result).

```python
phi0 = np.pi/180 * 20.  # convert initial phi to radians
t_pts, phi, phidot = solve_for_phi(phi0, t_max=15.)
```

```python
fig = plt.figure()
ax = fig.add_subplot(1,1,1)
ax.plot(t_pts, 180./np.pi * phi)
fig.tight_layout()  # make the spacing of subplots nicer

```

Ok, now we need an ode function for the small angle approximation.  It's very easy now to copy and modify our other function!

```python
def ode_rhs_small_angle(u_vec, t, *params):
    """
    Right-hand side (rhs) of the differential equation, with
    u_vec = [\phi, \dot\phi] and params = [g, R].  Returns the list of
    d(u_vec)/dt, as prescribed by the differential equation.

    """
    phi, phidot = u_vec  # We don't actually use x or y here, but could!
    g, R = params
    return [phidot, -g*phi/R]
```

And we can put them together into one solver function:

```python
def solve_for_phi_all(phi0, phidot0=0, t_min=0., t_max=1., g=9.8, R=5.):
    """
    Solve the differential equation for the skateboard Example 1.2 in Taylor
    using the exact equation and the small angle approximation.
    The result for t, \phi(t) and \dot\phi(t) are returned for a grid with
    t_min < t < t_max and a hardwired (for now) spacing of 0.01 seconds.
    The ODE solver is odeint from scipy, with specified tolerances.
    Units are mks and angles are in radians.
    """

    # absolute and relative tolerances for ode solver
    abserr = 1.0e-8
    relerr = 1.0e-6

    # initial conditions for [phi, phidot]
    u0_vec = [phi0, phidot0]

    t_pts = np.arange(t_min, t_max, 0.01)

    # Integrate the differential equations
    phi, phidot = odeint(ode_rhs_exact, u0_vec, t_pts, args=(g, R),
                     atol=abserr, rtol=relerr).T
    phi_sa, phidot_sa = odeint(ode_rhs_small_angle, u0_vec, t_pts, args=(g, R),
                     atol=abserr, rtol=relerr).T

    return t_pts, phi, phidot, phi_sa, phidot_sa
```

Always try it out!

```python
phi0 = np.pi/180 * 20.
t_pts, phi, phidot, phi_sa, phidot_sa = solve_for_phi_all(phi0, t_max=15.)

```

```python
fig = plt.figure()
ax = fig.add_subplot(1,1,1)
ax.plot(t_pts, 180./np.pi * phi)
ax.plot(t_pts, 180./np.pi * phi_sa)
fig.tight_layout()  # make the spacing of subplots nicer

```

This is actually the plot that is requested, so we could analyze it at this stage, but instead let's improve the plot and see how to save it.

### Ok, now for some more systematic plotting

Here we see examples of applying limits to the x and y axes as well as labels and a title.

```python
fig = plt.figure(figsize=(8,6))
ax = fig.add_subplot(1,1,1)
ax.set_xlim(0.,15.)
ax.set_ylim(-25.,25.)
ax.set_xlabel('t (sec)')
ax.set_ylabel(r'$\phi$')
ax.set_title(r'$\phi_0 = 20$ degrees')
line_exact, = ax.plot(t_pts, 180./np.pi * phi, label='exact')
line_sa, = ax.plot(t_pts, 180./np.pi * phi_sa, label='small angle')
ax.legend()

# save the figure
fig.savefig('Taylor_prob_1.50.png', bbox_inches='tight')
```

### Bonus: repeat with widgets!

This actually generalizes problems 1.50 and 1.51 so that you can examine any angle in between.  Use it to check your figure for 1.51.

```python
from ipywidgets import interact, fixed
import ipywidgets as widgets

def rad_to_deg(theta_rad):
    """Take as input an angle in radians and return it in degrees."""
    return 180./np.pi * theta_rad

def deg_to_rad(theta_deg):
    """Take as input an angle in degrees and return it in radians."""
    return np.pi/180. * theta_deg

```

```python
def plot_exact_and_small_angle(phi0_deg=0):
    phi0_rad = deg_to_rad(phi0_deg)
    t_pts, phi_rad, phidot, phi_sa_rad, phidot_sa = \
         solve_for_phi_all(phi0_rad, t_max=15.)
    phi_deg = rad_to_deg(phi_rad)
    phi_sa_deg = rad_to_deg(phi_sa_rad)

    fig = plt.figure(figsize=(8,6))
    ax = fig.add_subplot(1,1,1)
    line_exact, = ax.plot(t_pts, phi_deg, label='exact')
    line_sa, = ax.plot(t_pts, phi_sa_deg, label='small angle')
    ax.legend()
    ax.set_xlim(0.,15.)
    #ax.set_ylim(-90.,90.)
    ax.set_xlabel('t (sec)')
    ax.set_ylabel(r'$\phi$')
    ax.set_title(fr'$\phi_0 = {phi0_deg:.0f}$')
    plt.show()

```

```python
interact(plot_exact_and_small_angle, phi0_deg=(0.,90.));
```

```python
# to avoid the jiggling and do some formatting
phi0_deg_widget = widgets.FloatSlider(min=0., max=120.0, step=0.1, value=0.,
                                     description=r'$\phi_0$ (degrees)',
                                     readout_format='.0f',
                                     continuous_update=False
                                    )
interact(plot_exact_and_small_angle, phi0_deg=phi0_deg_widget);
```

```python

```