---
author: Nathan Carter (ncarter@bentley.edu)
---

This answer assumes you have imported SymPy as follows.

```python
from sympy import *                   # load all math functions
init_printing( use_latex='mathjax' )  # use pretty math output
```

The following code tells SymPy that $x$ is a variable and that
$y$ is a function of $x$.  It then expresses $\frac{dy}{dx}$ as the
derivative of $y$ with respect to $x$.

```python
var( 'x' )                 # Let x be a variable.
y = Function('y')(x)       # Literally, y is a function, named y, based on x.
dydx = Derivative( y, x )  # How to write dy/dx.
dydx                       # Let's see how SymPy displays dy/dx.
```

$\displaystyle \frac{d}{d x} y{\left(x \right)}$

Let's now write a very simple differential equation, $\frac{dy}{dx}=y$.

As with how to do implicit differentiation, SymPy expects us to move everything
to the left hand side of the equation.  In this case, that makes the equation
$\frac{dy}{dx}-y=0$, and we will use just the left-hand side to express our ODE.

```python
ode = dydx - y
ode
```

$\displaystyle - y{\left(x \right)} + \frac{d}{d x} y{\left(x \right)}$

database/tasks/How to write an ordinary differential equation/Python, using SymPy.ipynb


# Kaczmarz Algorithm

Given $Ax = b$, iteratively solve with the update:
\begin{equation}
x^{(k+1)} = x^{(k)} + \frac{b_i - \langle a_i , x^{(k)}\rangle}{\lvert a_i \rvert^2}a_i^*
\end{equation}
where $i = k \mathrm{mod} m +1$

```python
import numpy as np
from numpy import random
from numpy import linalg as LA
from collections import OrderedDict

def relative_error_to_reference(x, x_ref):
    return LA.norm(x - x_ref) / LA.norm(x_ref)

def cg(A, x, b, tol=1.0e-08, max_it=200):
    # Iteration counter
    k = 0
    # Residual
    r = b - np.einsum('ij,j', A, x)
    # Search direction
    p = r
    while LA.norm(r) > tol:
        tmp = np.einsum('i,i', r, r)
        alpha = tmp / np.einsum('i,ij,j', p, A, p)
        x = x + alpha * p
        # Update residual
        r = r - alpha * np.einsum('ij,j', A, p)
        # Update search direction
        beta = np.einsum('i,i', r, r) / tmp
        p = r + beta * p
        #print('Iteration {}, Norm of residual {}'.format(k, LA.norm(r)))
        k += 1
        if k >= max_it:
            raise RuntimeError('Maximum number of iterations reached!')
            break
    return k, LA.norm(r), x

def kaczmarz(A, x, b, tol=1.0e-08, max_it=2000):
    m = A.shape[0]
    # Iteration counter
    k = 0
    # Residual
    r = b - np.einsum('ij,j', A, x)
    while LA.norm(r) > tol:
        i = k%m
        alpha = (b[i] - np.einsum('i,i', A[i,:], x)) / np.einsum('i,i', A[i,:], A[i,:])
        x = x + alpha * np.transpose(A[i,:])
        r = b - np.einsum('ij,j', A, x)
        #for i in range(m):
        #    j = m-i-1
        #    alpha = (b[j] - np.einsum('i,i', A[j, :], x)) / np.einsum('i,i', A[j, :], A[j, :])
        #    x = x + alpha * np.conjugate(A[j, :])
        #    r = b - np.einsum('ij,j', A, x)
        #print('Iteration {}, Norm of residual {}'.format(k, LA.norm(r)))
        k += 1
        if k >= max_it:
            raise RuntimeError('Maximum number of iterations reached!')
            break
    return k, LA.norm(r), x

def random_kaczmarz(A, x, b, tol=1.0e-08, max_it=20000):
    m = A.shape[0]
    # Iteration counter
    k = 0
    # Residual
    r = b - np.einsum('ij,j', A, x)
    # Random start row index
    i = random.randint(0, m)
    # 2-norm squared of row
    norm = np.einsum('i,i', A[i, :], A[i, :])
    while LA.norm(r) > tol:
        # Do a Kaczmarz sweep on row i
        alpha = (b[i] - np.einsum('i,i', A[i, :], x)) / np.einsum('i,i', A[i, :], A[i, :])
        x = x + alpha * np.conjugate(A[i, :])
        r = b - np.einsum('ij,j', A, x)
        #print('Iteration {}, Norm of residual {}'.format(k, LA.norm(r)))
        # Propose random index
        prop_i = random.randint(0,m)
        # 2-norm squared of row
        prop_norm = np.einsum('i,i', A[prop_i, :], A[prop_i, :])
        acceptance = min(1.0, prop_norm / norm)
        if random.uniform() < acceptance:
            i = prop_i
            norm = prop_norm
        k += 1
        if k >= max_it:
            raise RuntimeError('Maximum number of iterations reached!')
            break
    return k, LA.norm(r), x

dim = 10
M = np.random.randn(dim, dim)
# Make sure our matrix is SPD
A = 0.5 * (M + M.transpose())
A = A * A.transpose()
A += dim * np.eye(dim)
b = np.random.rand(dim)
x_ref = LA.solve(A, b)

x_0 = np.zeros_like(b)

C = np.arange(6).reshape(2,3)
for i in C:
    print(i)

methods = OrderedDict({
    'Conjugate gradient' : cg,
    'Kaczmarz' : kaczmarz,
    'Random Kaczmarz' : random_kaczmarz
     })
#reports = OrderedDict()
for k, v in methods.items():
    print('@ {:s} algorithm'.format(k))
    it, residual, x = v(A, x_0, b)
    rel_err = relative_error_to_reference(x, x_ref)
    print('{:s} converged in {:d} iterations with relative error to reference {:.5E}\n'.format(k, it, rel_err))
    #reports.update({k: report})
```

    [0 1 2]
    [3 4 5]
    @ Conjugate gradient algorithm
    Conjugate gradient converged in 9 iterations with relative error to reference 2.10003E-09

    @ Kaczmarz algorithm
    Kaczmarz converged in 174 iterations with relative error to reference 7.04794E-09

    @ Random Kaczmarz algorithm
    Random Kaczmarz converged in 480 iterations with relative error to reference 8.09928E-09

```python

```


## Optimal Charging Example

We have an electric storage device with state-of-charge (SOC) $q_t \in \mathbb{R}_+$ at time $t$ and capacity $Q \in \mathbb{R}_+$. We denote the amount of energy charged from time $t$ to time $t+1$ as $u_t \in \mathbb{R}$, i.e., $q_{t+1} = q_t + u_t$. Power is limited by $C \in \mathbb{R}_+$ ($D \in \mathbb{R}_+$), the maximum possible magnitude of charging (discharging) power. The energy price $P(u_t)$ is higher when buying energy from the grid compared to the case of selling energy to the grid. Specifically,

\begin{equation}
P(u_i) = \begin{cases}
p_t u_t (1+\eta) \quad &\text{if} \quad u_t > 0 \\
p_t u_t (1-\eta) \quad &\text{otherwise},
\end{cases}
\end{equation}

where $p_t \in \mathbb{R}_+$ is the average market price at time $t$ and $0 < \eta < 1$. To optimize the cost of charging the energy storage from empty to full within a time period of length $T$, we solve the optimization problem

\begin{equation}
\begin{array}{ll}
\text{minimize} \quad & \sum_{t=0}^T p_t \left(u_t + \eta |u_t|\right) + \gamma u_t^2\\
\text{subject to} \quad &q_{t+1} = q_t + u_t \quad \forall t \in \{0,...,T \}\\
&-D \leq u_t \leq C \quad \forall t \in \{0,...,T \}\\
&0 \leq q_t \leq Q \quad \forall t \in \{0,...,T \}\\
&q_0 = 0\\
&q_{T+1} = Q,
\end{array}
\end{equation}

where $u_t \in \mathbb{R}$ and $q_t \in \mathbb{R}_+$ are the variables. We have added the regularization term $\gamma u_t^2$ to reduce stress on the electronic system due to peak power values, with $\gamma \in \mathbb{R}_+$. We reformulate the problem to be [DPP-compliant](https://www.cvxpy.org/tutorial/advanced/index.html#disciplined-parametrized-programming) by introducing the parameter $s_t = p_t \eta$ and we use time vectors $u \in \mathbb{R}^T$, $p, s \in \mathbb{R}_+^T$ and $q \in \mathbb{R}_+^{T+1}$ to summarize the temporal variables and parameters. Finally, we solve

\begin{equation}
\begin{array}{ll}
\text{minimize} \quad & p^T u + s^T |u| + \gamma \Vert u \Vert_2^2\\
\text{subject to} \quad &q_{1:T+1} = q_{0:T} + u\\
&-D \mathbb{1} \leq u \leq C \mathbb{1}\\
&\mathbb{0} \leq q \leq Q \mathbb{1}\\
&q_0 = 0\\
&q_{T+1} = Q,
\end{array}
\end{equation}

where $|u|$ is the element-wise absolute value of $u$. Let's define the corresponding CVXPY problem. To model a one-day period with a resolution of one minute, we choose $T=24 \cdot 60 = 1440$.

```python
import cvxpy as cp
import numpy as np

# define dimension
T = 1440

# define variables
u = cp.Variable(T, name='u')
q = cp.Variable(T+1, name='q')

# define parameters
p = cp.Parameter(T, nonneg=True, name='p')
s = cp.Parameter(T, nonneg=True, name='s')
D = cp.Parameter(nonneg=True, name='D')
C = cp.Parameter(nonneg=True, name='C')
Q = cp.Parameter(nonneg=True, name='Q')
gamma = cp.Parameter(nonneg=True, name='gamma')

# define objective
objective = cp.Minimize(p@u + s@cp.abs(u) + gamma*cp.sum_squares(u))

# define constraints
constraints = [q[1:] == q[:-1] + u,
               -D <= u, u<= C,
               0 <= q, q <= Q,
               q[0] == 0, q[-1] == Q]

# define problem
problem = cp.Problem(objective, constraints)
```

Assign parameter values and solve the problem. The one-day period starts at 2pm with a medium energy price level until 5pm, high price level from 5pm to midnight and low prices otherwise.

```python
import matplotlib.pyplot as plt

p.value = np.concatenate((3*np.ones(3*60),
                          5*np.ones(7*60),
                          1*np.ones(14*60)), axis=0)
eta = 0.1
s.value = eta*p.value
Q.value = 1
C.value = 3*Q.value/(24*60)
D.value = 2*C.value
gamma.value = 100

val = problem.solve()

fig, ax1 = plt.subplots()

ax1.plot(100*q.value, color='b')
ax1.grid()
ax1.set_xlabel('Time [min]')
ax1.set_ylabel('SOC [%]', color='b')
ax1.tick_params(axis='y', labelcolor='b')

ax2 = ax1.twinx()
ax2.plot(100*p.value / max(p.value), color='m')
ax2.set_ylabel('Price Level [%]', color='m')
ax2.tick_params(axis='y', labelcolor='m')
```

We observe that it is optimal to charge the storage with maximum power during the medium price phase, then empty the storage when prices are highest, and then fully charge the storage for the lowest price of the day. Generating C source for the problem is as easy as:

```python
from cvxpygen import cpg

cpg.generate_code(problem, code_dir='charging_code')
```

Now, you can use a python wrapper around the generated code as a custom CVXPY solve method.

```python
from charging_code.cpg_solver import cpg_solve
import numpy as np
import pickle
import time

# load the serialized problem formulation
with open('charging_code/problem.pickle', 'rb') as f:
    prob = pickle.load(f)

# assign parameter values
prob.param_dict['p'].value = np.concatenate((3*np.ones(3*60),
                                             5*np.ones(7*60),
                                             1*np.ones(14*60)), axis=0)
eta = 0.1
prob.param_dict['s'].value = eta*prob.param_dict['p'].value
prob.param_dict['Q'].value = 1
prob.param_dict['C'].value = 5*prob.param_dict['Q'].value/(24*60)
prob.param_dict['D'].value = 2*prob.param_dict['C'].value

# solve problem conventionally
t0 = time.time()
# CVXPY chooses eps_abs=eps_rel=1e-5, max_iter=10000, polish=True by default,
# however, we choose the OSQP default values here, as they are used for code generation as well
val = prob.solve(solver='OSQP', eps_abs=1e-3, eps_rel=1e-3, max_iter=4000, polish=False)
t1 = time.time()
print('\nCVXPY\nSolve time: %.3f ms' % (1000 * (t1 - t0)))
print('Objective function value: %.6f\n' % val)

# solve problem with C code via python wrapper
prob.register_solve('CPG', cpg_solve)
t0 = time.time()
val = prob.solve(method='CPG')
t1 = time.time()
print('\nCVXPYgen\nSolve time: %.3f ms' % (1000 * (t1 - t0)))
print('Objective function value: %.6f\n' % val)
```

\[1\] Wang, Yang, Brendan O'Donoghue, and Stephen Boyd. "Approximate dynamic programming via iterated Bellman inequalities." International Journal of Robust and Nonlinear Control 25.10 (2015): 1472-1496.


# Linear Algebra

## 1.1 Graph vector $\vec{a}$

\begin{align}
\vec{a} = \begin{bmatrix} 3 \\ 2 \end{bmatrix}
\end{align}

```python
import matplotlib.pyplot as plt

plt.arrow(0, 0, 3, 2, width=0.04)
plt.xlim(0,4)
plt.ylim(0,4);
```

## 1.2 Find $||\vec{b}||$. What does the norm of a vector represent?
\begin{align}
\vec{b} = \begin{bmatrix} 17 & -4 & -2 & 1\end{bmatrix}
\end{align}

```python
import numpy as np

norm = np.linalg.norm(np.array([17, -4, -2, 1]))
print(norm)
```

    17.60681686165901

Norm represents the magnitude/factor/distance of a vector.

## 1.3 Find $\vec{c} \cdot \vec{d}$

\begin{align}
\vec{c} = \begin{bmatrix}3 & 7 & -2 & 12\end{bmatrix}
\qquad
\vec{d} = \begin{bmatrix}9 & -7 & 4 & 6\end{bmatrix}
\end{align}

```python
c = np.array([3,  7, -2, 12])
d = np.array([9, -7,  4,  6])

print(np.dot(c, d))
```

    42

## 1.4 Find $E^{-1}$ and $E^{T}$

\begin{align}
E =
\begin{bmatrix}
    7 & 4 & 2 \\
    1 & 3 & -1 \\
    2 & 6 & -4
\end{bmatrix}
\end{align}

```python
E = np.array([[7, 4,  2],
              [1, 3, -1],
              [2, 6, -4]])

print(np.linalg.inv(E))
print(E.T)
```

    [[ 0.17647059 -0.82352941  0.29411765]
     [-0.05882353  0.94117647 -0.26470588]
     [ 0.          1.         -0.5       ]]
    [[ 7  1  2]
     [ 4  3  6]
     [ 2 -1 -4]]

# Intermediate Linear Algebra

## 2.1 Suppose that the number of customers at a ski resort as well as the number of inches of fresh powder (snow)  was recorded for 7 days.

### Customers: [820, 760, 1250, 990, 1080, 1450, 1600]

### Inches of new snow: [0, 1, 7, 1, 0, 6, 4 ]

## Find the mean, variance, and standard deviation for both the number of customers and inches of new snow for the week. You may use library functions, dataframes, .describe(), etc.

```python
import pandas as pd

customers = [820, 760, 1250, 990, 1080, 1450, 1600]
snow = [0, 1, 7, 1, 0, 6, 4]

df = pd.DataFrame({'customers': customers, 'snow': snow})

df.head()
```

      customers
      snow

      0
      820
      0

      1
      760
      1

      2
      1250
      7

      3
      990
      1

      4
      1080
      0

```python
df.mean()
```

    customers    1135.714286
    snow            2.714286
    dtype: float64

```python
df.std(ddof=0)
```

    customers    290.951991
    snow           2.710524
    dtype: float64

```python
df.var(ddof=0)
```

    customers    84653.061224
    snow             7.346939
    dtype: float64

## 2.2 Are the variances of the number of customers and inches of snow comparable?
## Why or why not?

No, customers has a much greater variance.

## 2.3 Find the variance-covariance matrix for the number of customers and inches of snow at the ski resort.

```python
df.cov()
```

      customers
      snow

      customers
      98761.904762
      670.238095

      snow
      670.238095
      8.571429

# PCA

## 3.1 Standardize the data so that it has a mean of 0 and a standard deviation of 1. (You may use library functions)

We have included some code to get you started so that you don't get stuck on something that isn't standardizing the data or PCA.

This might be helpful:

<https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.StandardScaler.html>

```python
# Let me get you some data to start you off.
import pandas as pd

data = {"Country": ["England","Wales","Scotland","North Ireland"],
        "Cheese": [105,103,103,66],
        "Carcass_Meat": [245,227,242,267],
        "Other_Meat": [685, 803, 750, 586],
        "Fish": [147, 160, 122, 93],
        "Fats_and_Oils": [193, 235, 184, 209],
        "Sugars": [156, 175, 147, 139],
        "Fresh_Potatoes": [720, 874, 566, 1033],
        "Fresh_Veg": [253, 265, 171, 143],
        "Other_Veg": [488, 570, 418, 355],
        "Processed_Potatoes": [198, 203, 220, 187],
        "Processed_Veg": [360, 365, 337, 334],
        "Fresh_Fruit": [1102, 1137, 957, 674],
        "Cereals": [1472, 1582, 1462, 1494],
        "Beverages": [57,73,53,47],
        "Soft_Drinks": [1374, 1256, 1572, 1506],
        "Alcoholic Drinks": [375, 475, 458, 135],
        "Confectionery": [54, 64, 62, 41]}

df = pd.DataFrame(data)

# Look at the data
df.head()
```

      Country
      Cheese
      Carcass_Meat
      Other_Meat
      Fish
      Fats_and_Oils
      Sugars
      Fresh_Potatoes
      Fresh_Veg
      Other_Veg
      Processed_Potatoes
      Processed_Veg
      Fresh_Fruit
      Cereals
      Beverages
      Soft_Drinks
      Alcoholic Drinks
      Confectionery

      0
      England
      105
      245
      685
      147
      193
      156
      720
      253
      488
      198
      360
      1102
      1472
      57
      1374
      375
      54

      1
      Wales
      103
      227
      803
      160
      235
      175
      874
      265
      570
      203
      365
      1137
      1582
      73
      1256
      475
      64

      2
      Scotland
      103
      242
      750
      122
      184
      147
      566
      171
      418
      220
      337
      957
      1462
      53
      1572
      458
      62

      3
      North Ireland
      66
      267
      586
      93
      209
      139
      1033
      143
      355
      187
      334
      674
      1494
      47
      1506
      135
      41

```python
from sklearn.preprocessing import StandardScaler

df_zstand = StandardScaler().fit_transform(df.select_dtypes('number'))
```

## 3.2 Perform PCA on the data and graph Principal Component 1 against Principal Component 2. (You may use library functions)

This might be helpful:

<https://scikit-learn.org/stable/modules/generated/sklearn.decomposition.PCA.html>

```python
from sklearn.decomposition import PCA

pca = PCA(2).fit_transform(df_zstand)

print(pca)

plt.scatter(x=[_[0] for _ in pca], y=[_[1] for _ in pca]);
```

# Clustering

## 4.1 Use K-Means to cluster the following data and then graph your results. (You may use library functions)

We have included some code to get you started so that you don't get stuck on something that isn't standardizing clustering.

Prioritize calculating the clusters over graphing them.

Scikit-Learn K-Means Documentation:

<https://scikit-learn.org/stable/modules/generated/sklearn.cluster.KMeans.html>

```python
points = pd.read_csv('https://raw.githubusercontent.com/ryanleeallred/datasets/master/points.csv')
points.head()
```

      x
      y

      0
      -7.846803
      -3.421277

      1
      -3.554323
      -6.884729

      2
      -0.192822
      -9.671030

      3
      -6.401456
      -5.223972

      4
      -0.804026
      -9.704457

```python
from sklearn.cluster import KMeans

sum_of_squared_distances = []
xs = range(1,15)
for x in xs:
    km = KMeans(n_clusters=x)
    km = km.fit(points)
    sum_of_squared_distances.append(km.inertia_)
```

```python
plt.plot(list(range(1,15)), sum_of_squared_distances);
```

```python
labels = KMeans(4).fit(points).labels_
```

```python
fig, ax = plt.subplots()
ax.scatter(x='x', y='y', data=points, c=labels)
ax.set_aspect('equal');
```


```python
%matplotlib inline
```

```python
import numpy as np
import matplotlib.pyplot as plt
```

# Motivation

We can view pretty much all of **machine learning (ML)** (and this is one of many possible views) as an **optimisation** exercise. Our challenge in supervised learning is to find a function that maps the inputs of a certain system to its outputs. Since we don't have direct access to that function, we have to estimate it. We aim to find the *best* possible estimate. Whenever we use the word "best" in mathematics, we imply some kind of optimisation. Thus we either maximise some **performance function**, which increases for better estimates, or minimise some **loss function**, which decreases for better estimates. In general, we refer to the function that we optimise as the **objective function**.

There are elements of both science and art in the choice of performance/loss functions. For now let us focus on optimisation itself.

# Univariate functions

From school many of us remember how to optimise functions of a single scalar variable — **univariate** functions, such as, for example,
$$f(x) = -2x^2 + 6x + 9.$$

In Python we would define this function as

```python
def func(x): return -2. * x**2 + 6. * x + 9.
```

So we can pass values of $x$ to it as arguments and obtain the corresponding values $f(x)$ as the function's return value:

```python
func(0.)
```

    9.0

Whenever we are dealing with functions, it is always a good idea to visually examine their graphs:

```python
xs = np.linspace(-10., 10., 100)
fs = [func(x) for x in xs]
plt.plot(xs, fs, 'o');
```

Unsurprisingly (if we remember high school mathematics), the graph of our univariate **quadratic** (because the highest power of $x$ in it comes as $x^2$) function is a **parabola**. We are lucky: this function is **concave** — if we join any two points on its graph, the straight line joining them will always lie below the graph. For such functions we can usually find the **global optimum** (**minimum** or **maximum**, in this case the function has a single **global maximum**).

# Global versus local optima

We say **global** optimum, because a function may have multiple optima. All of them are called **local** optima, but only the largest maxima (the smallest minima) are referred to as **global**.

Consider the function
$$f(x) = x \cos(x).$$
It has numerous local minima and local maxima over $x \in \mathbb{R}$, but no global minimum/maximum:

```python
xs = np.linspace(-100., 100., 1000)
fs = xs * np.cos(xs)
plt.plot(xs, fs);
```

Now consider the function
$$f(x) = \frac{1}{x} \sin(x).$$
It has a single global maximum, two global minima, and infinitely many local maxima and minima.

```python
xs = np.linspace(-100., 100., 1000)
fs = (1./xs) * np.sin(xs)
plt.plot(xs, fs);
```

# High school optimisation

Many of us remember from school this method of optimising functions. For our function, say
$$f(x) = -2x^2 + 6x + 9,$$
find the function's derivative. If we forgot how to differentiate functions, we can look up the rules of differentiation, say, on Wikipedia. In our example, differentiation is straightforward, and yields
$$\frac{d}{dx}f(x) = -4x + 6.$$

However, if we have completely forgotten the rules of differentiation, one particular Python library — the one for doing symbolic maths — comes in useful:

```python
import sympy
x = sympy.symbols('x')
func_diff = sympy.diff(-2. * x**2 + 6. * x + 9, x)
func_diff
```

    -4.0*x + 6.0

Our next step is to find such $x$ (we'll call it $x_{\text{max}}$), at which this derivative becomes zero. This notation is somewhat misleading, because it is $f(x_{\text{max}})$ that is maximum, not $x_{\text{max}}$ itself; $x_{\text{max}}$ is the *location* of the function's maximum:
$$\frac{d}{dx}f(x_{\text{max}}) = 0,$$
i.e.
$$-4x_{\text{max}} + 6 = 0.$$
Hence the solution is

$$x_{\text{max}} = -6 / (-4) = 3/2 = 1.5$$

We could also use SymPy to solve the above equation:

```python
roots = sympy.solve(func_diff, x)
roots
```

    [1.50000000000000]

```python
x_max = roots[0]
```

In order to check that the value is indeed a local maximum and not a local minimum (and not a **saddle point**, look them up), we look at the second derivative of the function,
$$\frac{d^2}{dx^2}f(x_{\text{max}}) = -4.$$
Since this second derivative is negative at $x_{\text{max}}$, we are indeed looking at an (at least local) maximum. In this case we are lucky: this is also a global maximum. However, in general, it isn't easy to check mathematically whether an optimum global or not. This is one of the major challenges in optimisation.

Let us now find the value of the function at the maximum by plugging in $x_{\text{max}}$ into $f$:

$$f_{\text{max}} = f(x_{\text{max}}) = -2 x_{\text{max}}^2 + 6 x_{\text{max}} + 9 = -2 \cdot 1.5^2 + 6 \cdot 1.5 + 9 = 13.5.$$

```python
f_max = func(x_max)
f_max
```

    13.5000000000000

Let us label this maximum on the function's graph:

```python
xs = np.linspace(-10., 10., 100)
fs = [func(x) for x in xs]
plt.plot(xs, fs, 'o')
plt.plot(x_max, f_max, 'o', color='red')
plt.axvline(x_max, color='red')
plt.axhline(f_max, color='red');
```

# Multivariate functions

So far we have considered the optimisation of **real-valued** functions of a single real variable, i.e. $f: \mathbb{R} \rightarrow \mathbb{R}$.

However, most functions that we encounter in data science and machine learning are **multivariate**, i.e. $f: \mathbb{R}^n \rightarrow \mathbb{R}$. Moreover, some are also **multivalued**, i.e. $f: \mathbb{R}^n \rightarrow \mathbb{R}^m$.

(Note: univariate/multivariate refers to the function's argument, whereas single-valued/multi-valued to the function's output.)

Consider, for example, the following single-valued, multivariate function:
$$f(x_1, x_2) = -x_1^2 - x_2^2 + 6x_1 + 3x_2 + 9.$$

We could define it in Python as

```python
def func(x1, x2): return -x1**2 - x2**2 + 6.*x1 + 3.*x2 + 9.
```

Let's plot its graph. First, we need to compute the values of the function on a two-dimensional mesh grid:

```python
x1s, x2s = np.meshgrid(np.linspace(-100., 100., 100), np.linspace(-100., 100., 100))
fs = func(x1s, x2s)
np.shape(fs)
```

    (100, 100)

Then we can use the following code to produce a 3D plot:

```python
from mpl_toolkits.mplot3d import Axes3D
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
ax.contour3D(x1s, x2s, fs, 50);
```

It may be more convenient to implement multivariate functions as functions of a single vector (more precisely, rank-1 NumPy array) in Python:

```python
def func(x): return -x[0]**2 - x[1]**2 + 6.*x[0] + 3.*x[1] + 9.
```

# Optimising multivariate functions analytically

The analytical method of finding the optimum of a multivariate function is similar to that for univariate functions. As the function has mutliple arguments, we need to find its so-called **partial derivative** with respect to each argument. They are computed similarly to normal derivatives, while pretending that all the other arguments are constants:
$$\frac{\partial}{\partial x_1} f(x_1, x_2) = -2x_1 + 6,$$
$$\frac{\partial}{\partial x_2} f(x_1, x_2) = -2x_2 + 3.$$

We call the vector of the function's partial derivatives its **gradient** vector, or **grad**:
$$\nabla f(x_1, x_2) = \begin{pmatrix} \frac{\partial}{\partial x_1} f(x_1, x_2) \\ \frac{\partial}{\partial x_2} f(x_1, x_2) \end{pmatrix}.$$

When the function is continuous and differentiable, all the partial derivatives will be 0 at a local maximum or minimum point. Saying that all the partial derivatives are zero at a point, $(x_1^*, x_2^*)$, is the same as saying the gradient at that point is the zero vector:
$$\nabla f(x_1^*, x_2^*) = \begin{pmatrix} \frac{\partial}{\partial x_1} f(x_1^*, x_2^*) \\ \frac{\partial}{\partial x_2} f(x_1^*, x_2^*) \end{pmatrix} = \begin{pmatrix} 0 \\ 0 \end{pmatrix} = \mathbf{0}.$$

In our example, we can easily establish that the gradient vector is zero at $x_1^* = 3$, $x_2^* = 1.5$. And the maximum value that is achieved is

```python
func([3, 1.5])
```

    20.25

```python
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
ax.contour3D(x1s, x2s, fs, 50)
ax.plot([3], [1.5], [20.25], 'o', color='red', markersize=20);
```

# The Jacobian

Notice that, for multivalued (not just multivariate) functions, $\mathbb{R}^n \rightarrow \mathbb{R}^m$, the **gradient** vector of partial derivatives generalises to the **Jacobian** matrix:
$$\mathbf{J} = \begin{pmatrix} \frac{\partial f_1}{\partial x_1} & \frac{\partial f_1}{\partial x_2} & \cdots & \frac{\partial f_1}{\partial x_n} \\ \vdots & \vdots & \ddots & \vdots \\ \frac{\partial f_m}{\partial x_1} & \frac{\partial f_m}{\partial x_2} & \cdots & \frac{\partial f_m}{\partial x_n} \end{pmatrix}.$$

# Newton-Raphson's method

**Newton-Raphson's method** is a numerical procedure for finding zeros (**roots**) of functions.

For example, consider again the function
$$f(x) = -2x^2 + 6x + 9.$$

```python
def func(x): return -2. * x**2 + 6. * x + 9.
```

We have already found that its derivative is given by
$$\frac{df}{dx}(x) = -4x + 6.$$

```python
def func_diff(x): return -4. * x + 6.
```

The Newton-Raphson method starts with some initial guess, $x_0$, and then proceeds iteratively:

$$x_{n+1} = x_n - \frac{f(x_n)}{\frac{d}{dx}f(x_n)}$$

Let's code it up:

```python
def newton_raphson_method(f, fdiff, x0, iter_count=10):
    x = x0
    print('x_0', x0)
    for i in range(iter_count):
        x = x - f(x) / fdiff(x)
        print('x_%d' % (i+1), x)
    return x
```

Now let's apply it to our function:

```python
newton_raphson_method(func, func_diff, -5.)
```

    x_0 -5.0
    x_1 -2.269230769230769
    x_2 -1.280023547880691
    x_3 -1.1040302676228468
    x_4 -1.0980830182607666
    x_5 -1.0980762113622329
    x_6 -1.098076211353316
    x_7 -1.098076211353316
    x_8 -1.098076211353316
    x_9 -1.098076211353316
    x_10 -1.098076211353316

    -1.098076211353316

We see that the method converges quite quickly to (one of the) roots. Notice that, which of the two roots we converge to depends on the initial guess:

```python
newton_raphson_method(func, func_diff, x0=5.)
```

    x_0 5.0
    x_1 4.214285714285714
    x_2 4.1005639097744355
    x_3 4.098077401218985
    x_4 4.098076211353589
    x_5 4.098076211353316
    x_6 4.098076211353316
    x_7 4.098076211353316
    x_8 4.098076211353316
    x_9 4.098076211353316
    x_10 4.098076211353316

    4.098076211353316

**Newton-Raphson** is a **root finding**, not an **optimisation**, algorithm. However, recall that optimisation is equivalent to finding the root of the derivative function. Thus we can apply this algorithm to the derivative function (we also need to provide the second derivative function) to find a local optimum of the function:

```python
def func_diff2(x): return -4.
```

```python
newton_raphson_method(func_diff, func_diff2, -5.)
```

    x_0 -5.0
    x_1 1.5
    x_2 1.5
    x_3 1.5
    x_4 1.5
    x_5 1.5
    x_6 1.5
    x_7 1.5
    x_8 1.5
    x_9 1.5
    x_10 1.5

    1.5

The result is consistent with our analytical solution.

# Newton's method for multivariate functions

Newton's method can be generalised to mutlivariate functions. For multivalued multivariate functions $f: \mathbb{R}^k \rightarrow \mathbb{R}^k$, the method becomes
$$x_{n+1} = x_n - \mathbf{J}(x_n)^{-1} f(x_n),$$
where $\mathbf{J}$ is the Jacobian.

Since inverses are only defined for square matrices, for functions $f: \mathbb{R}^k \rightarrow \mathbb{R}^m$, we use the Moore-Penrose pseudoinverse $\mathbf{J}^+ = (\mathbf{J}^T \mathbf{J})^{-1} \mathbf{J}^T$ instead of $\mathbf{J}^{-1}$. Let's code this up.

Inside our generalised implementation of Newton-Raphson, we'll be working with vectors. It's probably a good idea to assume that the function and the Jacobian return rank-2 NumPy arrays.

However, one may have coded up the function as

```python
def func(x): return -x[0]**2 - x[1]**2 + 6.*x[0] + 3.*x[1] + 9.
```

and the Jacobian as

```python
def func_diff(x): return np.array([-2.*x[0] + 6., -2.*x[1] + 3.])
```

Let's see how we can convert NumPy stuff to rank-2 arrays. For rank-1 arrays:

```python
a = np.array([3., 5., 7.])
np.reshape(a, (np.shape(a)[0], -1))
```

    array([[ 3.],
           [ 5.],
           [ 7.]])

if we want a column (rather than row) vector, which is probably a sensible default. If we wanted a row vector, we could do

```python
np.reshape(a, (-1, np.shape(a)[0]))
```

    array([[ 3.,  5.,  7.]])

Existing rank-2 arrays remain unchanged by this:

```python
a = np.array([[1., 2., 3.], [4., 5., 6.], [7., 8., 9.]])
np.reshape(a, (np.shape(a)[0], -1))
```

    array([[ 1.,  2.,  3.],
           [ 4.,  5.,  6.],
           [ 7.,  8.,  9.]])

```python
np.reshape(a, (-1, np.shape(a)[0]))
```

    array([[ 1.,  2.,  3.],
           [ 4.,  5.,  6.],
           [ 7.,  8.,  9.]])

For scalars, `np.shape(a)[0]` won't work, as their shape is `()`, so we need to do something special. Based on this information, let us implement the auxiliary function `to_rank_2`:

```python
def to_rank_2(arg, row_vector=False):
    shape = np.shape(arg)
    size = 1 if len(shape) == 0 else shape[0]
    new_shape = (-1, size) if row_vector else (size, -1)
    return np.reshape(arg, new_shape)
```

And test it:

```python
to_rank_2(5.)
```

    array([[ 5.]])

```python
to_rank_2([1., 2., 3.])
```

    array([[ 1.],
           [ 2.],
           [ 3.]])

```python
to_rank_2([[1.], [2.], [3.]])
```

    array([[ 1.],
           [ 2.],
           [ 3.]])

```python
to_rank_2([[1., 2., 3.]])
```

    array([[ 1.,  2.,  3.]])

```python
to_rank_2([[1., 2., 3], [4., 5., 6.]])
```

    array([[ 1.,  2.,  3.],
           [ 4.,  5.,  6.]])

Now let's generalise our implementation of the Newton-Raphson method:

```python
def newton_raphson_method(f, fdiff, x0, iter_count=10):
    x = to_rank_2(x0)
    for i in range(iter_count):
        f_x = to_rank_2(f(x))
        fdiff_x = to_rank_2(fdiff(x), row_vector=True)
        non_square_jacobian_inv = np.dot(np.linalg.inv(np.dot(fdiff_x.T, fdiff_x)), fdiff_x.T)
        x = x - np.dot(non_square_jacobian_inv, f_x)
        print('x_%d' % (i+1), x)
    return x
```

```python
newton_raphson_method(func, func_diff, np.array([-10., -10.]), iter_count=5)
```

    x_1 [[-80.25 ]
     [ 25.125]]
    x_2 [[-80.25 ]
     [ 25.125]]
    x_3 [[-80.25 ]
     [ 25.125]]
    x_4 [[-80.25 ]
     [ 25.125]]
    x_5 [[-80.25 ]
     [ 25.125]]

    array([[-80.25 ],
           [ 25.125]])

```python
func_diff([-80.25, 25.125])
```

    array([ 166.5 ,  -47.25])

**NB! TODO: The above doesn't seem to work at the moment. The returned optimum is wrong. Can you spot a problem with the above implementation?**

# Quasi-Newton method

In practice, we may not always have access to the Jacobian of a function. There are numerical methods, known as **quasi-Newton methods**, which approximate the Jacobian numerically.

One such method is the **Broyden-Fletcher-Goldfarb-Shanno (BFGS)** algorithm. It is generally a bad idea to implement these algorithms by hand, since their implementations are often nuanced and nontrivial.

Fortunately, Python libraries provide excellent implementations of optimisation algorithms.

Let us use SciPy to optimise our function.

Remember that to maximise a function we simply minimise its negative, which is what we achieve with the Python lambda below:

```python
import scipy.optimize
scipy.optimize.minimize(lambda x: -func(x), np.array([-80., 25.]), method='BFGS')
```

          fun: -20.249999999999446
     hess_inv: array([[ 0.53711516,  0.13107271],
           [ 0.13107271,  0.96288482]])
          jac: array([  1.43051147e-06,  -4.76837158e-07])
      message: 'Optimization terminated successfully.'
         nfev: 32
          nit: 4
         njev: 8
       status: 0
      success: True
            x: array([ 3.0000007 ,  1.49999975])

# Grid search

What we have considered so far isn't the most straightforward optimisation procedure. A natural first thing to do is often the **grid search**.

In grid search, we pick a subset of the parameter search, usually a rectangular grid, evaluate the value at each grid point and pick the point where the function is largest (smallest) as the approximate location of the maximum (minimum).

As a by-product of the grid search we get a heat-map — an excellent way of visualising the magnitude of the function on the parameter space.

If we have more than two parameters, we can produce heatmaps for each parameter pair. (E.g., for a three-dimensional function, $(x_1, x_2)$, $(x_1, x_3)$, $(x_2, x_3)$.)

Grid search is often useful for **tuning** machine learning **hyperparameters** and finding optimal values for trading (and other) strategies, in which case a single evaluation of the objective function may correspond to a single backtest run over all available data.

Let us use the following auxiliary function from https://matplotlib.org/gallery/images_contours_and_fields/image_annotated_heatmap.html

```python
def heatmap(data, row_labels, col_labels, ax=None,
            cbar_kw={}, cbarlabel="", **kwargs):
    """
    Create a heatmap from a numpy array and two lists of labels.

    Arguments:
        data       : A 2D numpy array of shape (N,M)
        row_labels : A list or array of length N with the labels
                     for the rows
        col_labels : A list or array of length M with the labels
                     for the columns
    Optional arguments:
        ax         : A matplotlib.axes.Axes instance to which the heatmap
                     is plotted. If not provided, use current axes or
                     create a new one.
        cbar_kw    : A dictionary with arguments to
                     :meth:`matplotlib.Figure.colorbar`.
        cbarlabel  : The label for the colorbar
    All other arguments are directly passed on to the imshow call.
    """

    if not ax:
        ax = plt.gca()

    # Plot the heatmap
    im = ax.imshow(data, **kwargs)

    # Create colorbar
    cbar = ax.figure.colorbar(im, ax=ax, **cbar_kw)
    cbar.ax.set_ylabel(cbarlabel, rotation=-90, va="bottom")

    # We want to show all ticks...
    ax.set_xticks(np.arange(data.shape[1]))
    ax.set_yticks(np.arange(data.shape[0]))
    # ... and label them with the respective list entries.
    ax.set_xticklabels(col_labels)
    ax.set_yticklabels(row_labels)

    # Let the horizontal axes labeling appear on top.
    ax.tick_params(top=True, bottom=False,
                   labeltop=True, labelbottom=False)

    # Rotate the tick labels and set their alignment.
    plt.setp(ax.get_xticklabels(), rotation=-30, ha="right",
             rotation_mode="anchor")

    # Turn spines off and create white grid.
    for edge, spine in ax.spines.items():
        spine.set_visible(False)

    ax.set_xticks(np.arange(data.shape[1]+1)-.5, minor=True)
    ax.set_yticks(np.arange(data.shape[0]+1)-.5, minor=True)
    ax.grid(which="minor", color="w", linestyle='-', linewidth=3)
    ax.tick_params(which="minor", bottom=False, left=False)

    return im, cbar
```

```python
def func(x1, x2): return -x1**2 - x2**2 + 6.*x1 + 3.*x2 + 9.
x1s_ = np.linspace(-100., 100., 10)
x2s_ = np.linspace(-100., 100., 10)
x1s, x2s = np.meshgrid(x1s_, x2s_)
fs = func(x1s, x2s)
np.shape(fs)
heatmap(fs, x1s_, x2s_)[0];
```

# Random search

Sometimes a **random search** may be preferred over grid search. This also enables us to incorporate our guess — a prior distribution — of the location of the optimum, so we can sample the parameter points from that prior distribution and evaluate the values of the function at those points.

Both **grid search** and **random search** are the so-called **embarrassingly parallel** methods and are trivial to parallelise, either over multiple cores on a single machine or over a cluster/cloud.

In general, it is suboptimal to explore a hypercube of the parameter space by systematically going through each point in a grid. Sobol sequences give the optimal sequence of points to try — see Sergei Kucherenko's work in this area.

# Stochastic and batch gradient descent

When working with **aritificial neural networks (ANNs)** we usually prefer the **stochastic** and **batch gradient descent methods** over the quasi-Newton methods. We will examine these methods when we introduce ANNs.


```python
import sympy as sym #simbolica
import matplotlib.pyplot as plt #importa matplotlib solo pyplot
import matplotlib.image as mpimg
from IPython.display import Image #para importar y mostrar en jupyter imagenes

sym.init_printing() #activa a jupyter para mostrar simbolicamente el output
#%matplotlib widget
%matplotlib inline
```

```python
# Puente completo celda de carga es como se observa en la imagen.
Image(filename='straingauge.png',width=300)
```

```python
Image(filename='esq_sg.png',width=250)  #Circuito esquematico cambiando el enfoque
```

```python
#Se busca el equivalente theveniN
#Se van a plantear ecuaciones nodales
#Vth es el voltaje en Vo cuando Io=0 => CA

sym.var('R1, R2, R3, R4, Io, Vo, Vth')
sym.var('Va, Vb, Vp')
find=sym.Matrix(([Vo], [Va], [Vb])) #son las incognitas
ec_p_0=sym.Eq((Va-Vo)/R1+(Vb-Vo)/R2,0) # Nodo Vo ; Io=0
ec_p_1=sym.Eq((Va-Vb),Vp) #SuperNodo
ec_p_2=sym.Eq((Va/R3)+Vb/R4,0) # Tierra Io=0
display(sym.Eq(Vth,sym.factor(sym.simplify(sym.solve([ec_p_0,ec_p_1,ec_p_2],find)[Vo]))))
Vth=sym.solve([ec_p_0,ec_p_1,ec_p_2],find)[Vo]
```

```python
#Circuito para encontrar eq. Norton Io=-In Vo=0 -> CC
Image(filename='esq_sg2.png',width=250)
```

```python
#Se busca el equivalente Norton y se usa LKV
sym.var('I1,I2,In')
ec_p_3=sym.Eq(I1*R3+Vp+(I1-In)*R4,0)
ec_p_4=sym.Eq(I2*R1+(I2-In)*R2,Vp)
ec_p_5=sym.Eq((In-I1)*R4+(In-I2)*R2,0)
display(sym.Eq(In,sym.simplify(sym.factor(sym.solve([ec_p_3,ec_p_4,ec_p_5],(In,I1,I2))[In]))))
In=sym.solve([ec_p_3,ec_p_4,ec_p_5],(In,I1,I2))[In]
```

```python
#Por lo tanto La resistencia de Thevenin
sym.var('Rth')
display(sym.Eq(Rth,sym.simplify(sym.factor(Vth/In))))
Rth=sym.simplify(Vth/In)
```

```python
#Calculo de forma directa
Image(filename='esq_sg.png',width=250)  #Circuito esquematico cambiando el enfoque

```

```python
#De forma directa Vth es VR2+VR4
# Vth= Vp/(R1+R2) * R2 - Vp/(R3+R4) * R4
# Rth cuando Vp=0 es Rth=R1//R2 + R3//R4
sym.var('Vth_, Rth_')
display(sym.Eq(Vth_,sym.simplify(sym.factor(Vp*(R2/(R1+R2)-R4/(R3+R4))))))
Vth_=sym.fu((Vp/(R1+R2))*R2-(Vp/(R3+R4))*R4)

display(sym.Eq(Rth_,(R1**-1+R2**-1)**-1+(R3**-1+R4**-1)**-1))
display(sym.Eq(Rth_,sym.simplify(sym.factor((R1**-1+R2**-1)**-1+(R3**-1+R4**-1)**-1))))

Rth_=sym.factor(((R1**-1+R2**-1)**-1)+(R3**-1+R4**-1)**-1)

```


# Ejercicios Pérdida de Significancia

```python
import numpy as np
import matplotlib.pyplot as plt
```

## Comentario de la clase...

Del ejercicio realizado en clases teníamos problemas con $p=53$ para representación en *doble precisión*. Se puede representar hasta $2^{53}$ sin problemas y luego solo podemos sumar múltiplos de $2$ (combinaciones de los bits asociados a $2^{1}$, $2^{2}$, $2^{3}$, $\dots$), ya que se pierde el $2^0$. Ver el ejemplo en el video de la clase.

```python
2.**53, 2.**53+1, 2.**53+2, 2.**53+3, 2.**53+4, 2.**53+5, 2.**53+6, 2.**53+7, 2.**53+8, 2.**53+9
```

## Ejercicio 1

Obtener $a+\sqrt{a^2 + b^4}$, $a=-12345678987654321$, $b=123$.

Resultado aproximado: $9.2699089790398179 \times 10^{-9}$

```python
a = -12345678987654321.
b = 123.
```

```python
print("a: %f, b: %f" % (a, b))
```

Perdemos un dígito...

### Cálculo directo

```python
a + np.sqrt(a ** 2 + b ** 4)
```

### Manejo algebraico

\begin{equation}
    \begin{split}
        a+\sqrt{a^2 + b^4} & = a+\sqrt{a^2 + b^4} \cdot \frac{a-\sqrt{a^2 + b^4}}{a- \sqrt{a^2 + b^4}} \\
            & = \frac{a^2-a^2 - b^4}{a- \sqrt{a^2 + b^4}} \\
            & = \frac{-b^4}{a - \sqrt{a^2 + b^4}} \\
            & = \frac{b^4}{\sqrt{a^2 + b^4} - a}
    \end{split}
\end{equation}

```python
b ** 4 / (np.sqrt(a ** 2 + b ** 4) - a)
```

## Ejercicio 2

Obtener

\begin{equation}
    x^2 = 1.2222222^2 + 3344556600^2
\end{equation}

Esto sería:
\begin{equation}
    x = \sqrt{1.2222222^2 + 3344556600^2}
\end{equation}

El resultado es aproximadamente $3.34455660000000000022332214\times 10^9$

```python
c1 = 1.2222222
c2 = 3344556600.
```

```python
print(c1, c2)
```

```python
x1 = np.sqrt(c1 ** 2 + c2 ** 2)
```

```python
print(x1)
```

Utilizando

\begin{equation}
    x^2 = (c_1 + c_2)^2 - 2c_1c_2
\end{equation}

```python
x2 = np.sqrt((c1 + c2) ** 2 - 2 * c1 * c2)
```

```python
print(x2)
```

Utilizando un uno conveniente

\begin{equation}
    \begin{split}
        x^2 & = \sqrt{c_1^2 + c_2^2} \, \frac{\sqrt{c_1^2 + c_2^2}}{\sqrt{c_1^2 + c_2^2}} \\
            & = \frac{c_1^2 + c_2^2}{\sqrt{c_1^2 + c_2^2}}
    \end{split}
\end{equation}

```python
x3 = (c1 ** 2 + c2 ** 2) / np.sqrt(c1 ** 2 + c2 ** 2)
```

```python
print(x3)
```

```python
print(abs(3.34455660000000000022332214e9 - x1))
print(abs(3.34455660000000000022332214e9 - x2))
print(abs(3.34455660000000000022332214e9 - x3))
```

No todos los manejos algebraicos nos ayudarán a resolver problemas mal planteados.

## Ejercicio 3

Calcular las raíces de

\begin{equation}
    x^2 + 9^{12} x - 3 = 0
\end{equation}

\begin{equation}
    x = \frac{-9^{12} \pm \sqrt{9^{24} + 12}}{2}
\end{equation}

Una raíz sería:

$x_1=\frac{-9^{12} - \sqrt{9^{24} + 12}}{2}= −2.824\times 10^{11}$

En el caso de la segunda raíz esto es muy cercano a $0$

$x_2=\frac{-9^{12} + \sqrt{9^{24} + 12}}{2}$

$-9^{12} + \sqrt{9^{24}} \approx 0$ pero según cómo lo calculemos vamos a poder recuperar algunos decimales.

### Cálculo directo

```python
def root(a, b, c):
    r1 = (-b - np.sqrt(b ** 2 - 4 * a * c)) / (2 * a)
    r2 = (-b + np.sqrt(b ** 2 - 4 * a * c)) / (2 * a)
    return r1, r2
```

```python
a = 1.
b = 9. ** 12
c = -3.
```

```python
x1, x2 = root(a, b, c)
```

```python
print(x1, x2)
```

### Manejo algebraico

\begin{equation}
    \begin{split}
        x_2 & = \frac{-b + \sqrt{b^2 - 4ac}}{2a} \\
            & = \frac{-b + \sqrt{b^2 - 4ac}}{2a} \, \frac{b + \sqrt{b^2 - 4ac}}{b + \sqrt{b^2 - 4ac}} \\
            & = \frac{-4ac}{2a(b + \sqrt{b^2 - 4ac})} \\
            & = \frac{-2c}{(b + \sqrt{b^2 - 4ac})}
    \end{split}
\end{equation}

```python
def root_improved(a, b, c):
    r1 = (-b - np.sqrt(b ** 2 - 4 * a * c)) / (2 * a)
    r2 = -2 * c / (b + np.sqrt(b ** 2 - 4 * a * c))
    return r1, r2
```

```python
x1, x2 = root_improved(a, b, c)
```

```python
print(x1, x2)
```

Mirando con mayor detalle, si $4|ac| \ll b^2$, $b$ y $\sqrt{b^2-4ac}$ son similares en magnitud por lo tanto una de las raíces va a sufrir pérdida de significancia. Revisar el libro guía para más detalles de cómo solucionar el problema.

## Ejercicio 4

Sean
\begin{equation}
    E_1(x)=\frac{1-\cos(x)}{\sin^2(x)}, \quad E_2(x)=\frac{1}{1+\cos(x)}
\end{equation}

¿Son $E_1(x)$ y $E_2(x)$ iguales? ¿Qué ocurre al evaluarlas cerca de $0$?

Primero, ¿$E_1(x)=E_2(x)$?

\begin{equation}
    \begin{split}
        E_1(x) & =E_2(x) \\
        \frac{1-\cos(x)}{\sin^2(x)} & = \frac{1}{1+\cos(x)} \\
        (1-\cos(x))(1+\cos(x)) &= \sin^2(x) \\
        1-\cos^2(x) &= \sin^2(x) \\
        1 &= \sin^2(x) + \cos^2(x)
    \end{split}
\end{equation}

Efectivamente son iguales. Ahora veamos numéricamente qué ocurre al evaluarla cerca de $0$.

```python
E1 = lambda x: (1 - np.cos(x)) / np.sin(x) ** 2
E2 = lambda x: 1 / (1 + np.cos(x))
```

```python
xa, xb = -.5, .5
N = 101
x = np.linspace(xa, xb, N)
```

```python
x
```

```python
plt.figure(figsize=(13, 4))
plt.subplot(1, 2, 1)
plt.plot(x, E1(x), 'b.')
plt.xlabel(r"$x$")
plt.ylabel(r"$E_1(x)$")
plt.grid(True)
plt.subplot(1, 2, 2)
plt.plot(x, E2(x), 'r.')
plt.xlabel(r"$x$")
plt.ylabel(r"$E_2(x)$")
plt.grid(True)
plt.tight_layout()
plt.show()
```

Tenemos problemas al evaluar $0$ en el denominador de $E_1(x)$ puesto que se indetermina.

```python
x = np.logspace(-19,0,20)[-1:0:-1]
o1 = E1(x)
o2 = E2(x)

print("x,                 E1(x),             E2(x)")
for i in np.arange(len(x)):
    print("%1.15f, %1.15f, %1.15f" % (x[i], o1[i], o2[i]))
```

## Ejercicio 5

¿Cuándo habría problemas al evaluar $f(x)$ y $g(x)$?

\begin{equation}
    f(x) = \frac{1-(1-x)^3}{x}, \quad g(x)= \frac{1}{1+x}-\frac{1}{1-x}
\end{equation}

```python
f = lambda x: (1 - (1 - x) ** 3) / x
g = lambda x: 1 / (1 + x) - 1 / (1 - x)
```

```python
xa, xb = -2, 2
N = 101
x = np.linspace(xa, xb, N)
```

```python
plt.figure(figsize=(13, 4))
plt.subplot(1, 2, 1)
plt.plot(x, f(x), 'b.')
plt.xlabel(r"$x$")
plt.ylabel(r"$f(x)$")
plt.grid(True)
plt.subplot(1, 2, 2)
plt.plot(x, g(x), 'r.')
plt.xlabel(r"$x$")
plt.ylabel(r"$g(x)$")
plt.grid(True)
plt.tight_layout()
plt.show()
```

Es claro que para $f(x)$ el problema se encuentra cercano a $0$ mientras que para $g(x)$ el problema está en $-1$ y $1$. En el caso de $f(x)$ basta expandirla para obtener $f(x)=x^2-3x+3$. Si operamos sobre $g(x)$ se obtiene $g(x)=\frac{2x}{x^2-1}$, el cual contiene los mismos problemas.

```python
ff = lambda x: x ** 2 - 3 * x + 3
gg = lambda x: 2 * x / (x ** 2 - 1)
```

```python
plt.figure(figsize=(12, 4))
plt.subplot(1, 2, 1)
plt.plot(x, ff(x), 'b.', label=r"$f(x)$")
plt.grid(True)
plt.subplot(1, 2, 2)
plt.plot(x, gg(x), 'r.', label=r"$g(x)$")
plt.grid(True)
plt.tight_layout()
plt.show()
```

¿Existirá alguna forma de solucionar los problemas de $g(x)$?

# Referencias

* Sauer, T. (2006). Numerical Analysis Pearson Addison Wesley.
* https://github.com/tclaudioe/Scientific-Computing/blob/master/SC1/03_floating_point_arithmetic.ipynb <- Revisar este enlace que tiene ejemplos y comentarios interesantes!


# Poisson Distribution - Broken Machine

> This document is written in *R*.
>
> ***GitHub***: https://github.com/czs108

## Background

> Suppose the *mean* number of machine malfunctions per week, or rate of malfunctions, is **3.4**.

## Question A

> What's the probability of the machine *not* malfunctioning next week?

\begin{equation}
\lambda = 3.4
\end{equation}

\begin{equation}
\begin{split}
P(X = 0) &= \frac{e^{-3.4} \times {3.4}^{0}}{0!} \\
    &= e^{-3.4} \\
    &= 0.033
\end{split}
\end{equation}

Use the `dpois` function directly.

```R
dpois(x=0, lambda=3.4)
```

0.0333732699603261

Or use the `exp` function.

```R
exp(-3.4)
```

0.0333732699603261

## Question B

> What's the probability of the machine malfunctioning **3** times next week?

\begin{equation}
\begin{split}
P(X = 3) &= \frac{e^{-3.4} \times {3.4}^{3}}{3!} \\
    &= 0.2186
\end{split}
\end{equation}

```R
dpois(x=3, lambda=3.4)
```

0.218617167086776

## Question C

> What's the *expectation* and *variance* of the machine malfunctions?

\begin{equation}
E(X) = \lambda = 3.4
\end{equation}

\begin{equation}
Var(X) = \lambda = 3.4
\end{equation}

exercises/Poisson Distribution - Broken Machine.ipynb


# For today's code challenge you will be reviewing yesterdays lecture material. Have fun!

### if you get done early check out [these videos](https://www.3blue1brown.com/neural-networks).

# The Perceptron

The first and simplest kind of neural network that we could talk about is the perceptron. A perceptron is just a single node or neuron of a neural network with nothing else. It can take any number of inputs and spit out an output. What a neuron does is it takes each of the input values, multplies each of them by a weight, sums all of these products up, and then passes the sum through what is called an "activation function" the result of which is the final value.

I really like figure 2.1 found in this [pdf](http://www.uta.fi/sis/tie/neuro/index/Neurocomputing2.pdf) even though it doesn't have bias term represented there.

If we were to write what is happening in some verbose mathematical notation, it might look something like this:

\begin{align}
 y = sigmoid(\sum(weight_{1}input_{1} + weight_{2}input_{2} + weight_{3}input_{3}) + bias)
\end{align}

Understanding what happens with a single neuron is important because this is the same pattern that will take place for all of our networks.

When imagining a neural network I like to think about the arrows as representing the weights, like a wire that has a certain amount of resistance and only lets a certain amount of current through. And I like to think about the node itselef as containing the prescribed activation function that neuron will use to decide how much signal to pass onto the next layer.

# Activation Functions (transfer functions)

In Neural Networks, each node has an activation function. Each node in a given layer typically has the same activation function. These activation functions are the biggest piece of neural networks that have been inspired by actual biology. The activation function decides whether a cell "fires" or not. Sometimes it is said that the cell is "activated" or not. In Artificial Neural Networks activation functions decide how much signal to pass onto the next layer. This is why they are sometimes referred to as transfer functions because they determine how much signal is transferred to the next layer.

## Common Activation Functions:

# Implementing a Perceptron from scratch in Python

### Establish training data

```python
import numpy as np

np.random.seed(812)

inputs = np.array([
    [0, 0, 1],
    [1, 1, 1],
    [1, 0, 1],
    [0, 1, 1]
])

correct_outputs = [[0], [1], [1], [0]]
```

### Sigmoid activation function and its derivative for updating weights

```python
def sigmoid(x):
  return 1 / (1 + np.exp(-x))

def sigmoid_derivative(x):
  sx = sigmoid(x)
  return sx * (1 - sx)
```

## Updating weights with derivative of sigmoid function:

### Initialize random weights for our three inputs

```python
weights = np.random.rand(3,1)
weights
```

    array([[0.32659171],
           [0.59345002],
           [0.25569456]])

### Calculate weighted sum of inputs and weights

```python
weighted_sum = np.dot(inputs,weights)
```

### Output the activated value for the end of 1 training epoch

```python
activated_value = sigmoid(weighted_sum)
activated_value
```

    array([[0.56357763],
           [0.76418031],
           [0.64159331],
           [0.70038767]])

### take difference of output and true values to calculate error

```python
error = correct_outputs - activated_value
error
```

    array([[-0.56357763],
           [ 0.23581969],
           [ 0.35840669],
           [-0.70038767]])

```python
derivative = sigmoid_derivative(error)
derivative
```

    array([[0.23115422],
           [0.24655628],
           [0.24214035],
           [0.22168396]])

### Put it all together

```python
weights +=  np.dot(inputs,derivative)
```

```python

```


```python
from math_so.model import lotka_volterra
import numpy as np
from scipy.integrate import solve_ivp
import matplotlib.pyplot as plt
```

# Numerical Differentiation

Using a Taylor expansion of the (sufficiently often differentiable) function $f$ about $x$,

1. $f(x+h) = f(x) + h f'(x) + \frac{h^2}{2!} f''(x) + \frac{h^3}{3!} f'''(x) + \ldots, $
2. $f(x-h) = f(x) - h f'(x) + \frac{h^2}{2!} f''(x) - \frac{h^3}{3!} f'''(x) + \ldots, $
3. $f(x+2h) = f(x) + 2h f'(x) + 4\frac{h^2}{2!} f''(x) + 8\frac{h^3}{3!} f'''(x) + \ldots, $
4. $f(x-2h) = f(x) - 2h f'(x) + 4\frac{h^2}{2!} f''(x) - 8\frac{h^3}{3!} f'''(x) + \ldots, $

we can derive the following finite difference approximations to the derivative of $f$ at $x$.

## First-order finite difference
From (1) we obtain:
$$\begin{align*}f'_\text{forward}(x) &=\displaystyle \frac{f(x+h) - f(x)}{h}
\color{red}{-\frac{h}{2!}f''(x)-\frac{h^2}{3!}f'''(x)+\dots}\\
                      &=\displaystyle \frac{f(x+h) - f(x)}{h}  \color{red}{+\mathcal{O}(h)}
\end{align*}$$

From (2) we obtain:
$$\begin{align*}f'_\text{backward}(x) &=\displaystyle \frac{f(x) - f(x-h)}{h}
\color{red}{+\frac{h}{2!}f''(x)-\frac{h^2}{3!}f'''(x)+\dots}\\
                      &=\displaystyle \frac{f(x) - f(x-h)}{h}  \color{red}{+\mathcal{O}(h)}
\end{align*}$$

From (1) and (2) we obtain:
$$\begin{align*}f'_\text{central}(x) &=\displaystyle \frac{f(x+h) - f(x-h)}{2h}
\color{red}{-\frac{h^2}{3!}f'''(x)+\dots}\\
                      &=\displaystyle \frac{f(x+h) - f(x-h)}{2h}  \color{red}{+\mathcal{O}(h^2)}
\end{align*}$$

The error for the forward and backward difference method os of order $h$, for the central methos the error is of order $h^2$. (From the highest oder term in the dropped part of the taylor series. [[1](https://www.youtube.com/watch?v=ZJkGI5DZQv8&list=PLYdroRCLMg5OvLx1EtY1ByvveJeTEXQd_&index=18)][[2](https://www.youtube.com/watch?v=C2Wk-wiXLvE&list=PLYdroRCLMg5OvLx1EtY1ByvveJeTEXQd_&index=20)])

Here's an example of a function whose derivatives we know analytically:

```python
f = lambda x: np.log(x)
x0 = 3
df_ex = 1/x0      #f'(x0)
d2f_ex = -1/x0**2 #f''(x0)
```

```python
def fd_forward1(f, x, h):
    """fd_forward1 calculates the derivative of f at x
    with First-order forward finite difference.
    """
    return (f(x + h) - f(x))/h
def fd_backward1(f, x, h):
    """fd_backward1 calculates the derivative of f at x
    with First-order backward finite difference.
    """
    return (f(x)-f(x - h))/h
def fd_central1(f, x, h):
    """fd_central1 calculates the derivative of f at x
    with First-order central finite difference.
    """
    return (f(x + h) - f(x-h))/(2*h)
```

**Forward/Backward**: Let's check if the error really decreases by a factor of 2 as we halve $h$, until ultimately roundoff errors spoil the convergence:

```python
h = .1
for m in np.arange(0,24):
    df = fd_forward1(f, x0, h)
    err = np.abs(df - df_ex)
    if m > 0:
        print('h = {:.2E}, err = {:.2E}, fac = {:.2f}'.format(h, err, err_old/err))
    err_old = err
    h = h/2
```

    h = 5.00E-02, err = 2.75E-03, fac = 1.98
    h = 2.50E-02, err = 1.38E-03, fac = 1.99
    h = 1.25E-02, err = 6.93E-04, fac = 1.99
    h = 6.25E-03, err = 3.47E-04, fac = 2.00
    h = 3.13E-03, err = 1.73E-04, fac = 2.00
    h = 1.56E-03, err = 8.68E-05, fac = 2.00
    h = 7.81E-04, err = 4.34E-05, fac = 2.00
    h = 3.91E-04, err = 2.17E-05, fac = 2.00
    h = 1.95E-04, err = 1.09E-05, fac = 2.00
    h = 9.77E-05, err = 5.43E-06, fac = 2.00
    h = 4.88E-05, err = 2.71E-06, fac = 2.00
    h = 2.44E-05, err = 1.36E-06, fac = 2.00
    h = 1.22E-05, err = 6.78E-07, fac = 2.00
    h = 6.10E-06, err = 3.39E-07, fac = 2.00
    h = 3.05E-06, err = 1.70E-07, fac = 2.00
    h = 1.53E-06, err = 8.48E-08, fac = 2.00
    h = 7.63E-07, err = 4.26E-08, fac = 1.99
    h = 3.81E-07, err = 2.16E-08, fac = 1.97
    h = 1.91E-07, err = 1.06E-08, fac = 2.05
    h = 9.54E-08, err = 5.90E-09, fac = 1.79
    h = 4.77E-08, err = 5.90E-09, fac = 1.00
    h = 2.38E-08, err = 1.06E-08, fac = 0.56
    h = 1.19E-08, err = 1.24E-09, fac = 8.50

```python
h = .1
for m in np.arange(0,24):
    df = fd_backward1(f, x0, h)
    err = np.abs(df - df_ex)
    if m > 0:
        print('h = {:.2E}, err = {:.2E}, fac = {:.2f}'.format(h, err, err_old/err))
    err_old = err
    h = h/2
```

    h = 5.00E-02, err = 2.81E-03, fac = 2.02
    h = 2.50E-02, err = 1.40E-03, fac = 2.01
    h = 1.25E-02, err = 6.96E-04, fac = 2.01
    h = 6.25E-03, err = 3.48E-04, fac = 2.00
    h = 3.13E-03, err = 1.74E-04, fac = 2.00
    h = 1.56E-03, err = 8.68E-05, fac = 2.00
    h = 7.81E-04, err = 4.34E-05, fac = 2.00
    h = 3.91E-04, err = 2.17E-05, fac = 2.00
    h = 1.95E-04, err = 1.09E-05, fac = 2.00
    h = 9.77E-05, err = 5.43E-06, fac = 2.00
    h = 4.88E-05, err = 2.71E-06, fac = 2.00
    h = 2.44E-05, err = 1.36E-06, fac = 2.00
    h = 1.22E-05, err = 6.78E-07, fac = 2.00
    h = 6.10E-06, err = 3.39E-07, fac = 2.00
    h = 3.05E-06, err = 1.70E-07, fac = 2.00
    h = 1.53E-06, err = 8.49E-08, fac = 2.00
    h = 7.63E-07, err = 4.24E-08, fac = 2.00
    h = 3.81E-07, err = 2.15E-08, fac = 1.98
    h = 1.91E-07, err = 1.16E-08, fac = 1.86
    h = 9.54E-08, err = 5.74E-09, fac = 2.01
    h = 4.77E-08, err = 3.41E-09, fac = 1.68
    h = 2.38E-08, err = 8.07E-09, fac = 0.42
    h = 1.19E-08, err = 1.74E-08, fac = 0.46

**Central**: Let's check if the error really decreases by a factor of 4 as we halve $h$, until ultimately roundoff errors spoil the convergence:

```python
h = .1
for m in np.arange(0,24):
    df = fd_central1(f, x0, h)
    err = np.abs(df - df_ex)
    if m > 0:
        print('h = {:.2E}, err = {:.2E}, fac = {:.2f}'.format(h, err, err_old/err))
    err_old = err
    h = h/2
```

    h = 5.00E-02, err = 3.09E-05, fac = 4.00
    h = 2.50E-02, err = 7.72E-06, fac = 4.00
    h = 1.25E-02, err = 1.93E-06, fac = 4.00
    h = 6.25E-03, err = 4.82E-07, fac = 4.00
    h = 3.13E-03, err = 1.21E-07, fac = 4.00
    h = 1.56E-03, err = 3.01E-08, fac = 4.00
    h = 7.81E-04, err = 7.54E-09, fac = 4.00
    h = 3.91E-04, err = 1.88E-09, fac = 4.00
    h = 1.95E-04, err = 4.70E-10, fac = 4.00
    h = 9.77E-05, err = 1.17E-10, fac = 4.03
    h = 4.88E-05, err = 2.93E-11, fac = 3.99
    h = 2.44E-05, err = 8.79E-12, fac = 3.33
    h = 1.22E-05, err = 4.85E-12, fac = 1.81
    h = 6.10E-06, err = 4.85E-12, fac = 1.00
    h = 3.05E-06, err = 3.15E-11, fac = 0.15
    h = 1.53E-06, err = 6.79E-11, fac = 0.46
    h = 7.63E-07, err = 7.76E-11, fac = 0.88
    h = 3.81E-07, err = 7.76E-11, fac = 1.00
    h = 1.91E-07, err = 5.04E-10, fac = 0.15
    h = 9.54E-08, err = 7.76E-11, fac = 6.50
    h = 4.77E-08, err = 1.24E-09, fac = 0.06
    h = 2.38E-08, err = 1.24E-09, fac = 1.00
    h = 1.19E-08, err = 8.07E-09, fac = 0.15

```python

```

```python
fig, ax = plt.subplots(figsize=(8, 5))
f = lambda x: np.log(x)
x0 = 0.2
df = lambda x: 1/x
h = .1
a,b = x0-h,x0+h

x = np.linspace(a-.03,b+.03,200)
y = f(x)
y0 = f(x0)
yt = df(x0) * (x - x0) + y0
ytf = fd_forward1(f, x0, h) * (x - x0) + y0
ytb = fd_backward1(f, x0, h) * (x - x0) + y0
ytc = fd_central1(f, x0, h) * (x - x0) + y0

ax.plot(x,y,'r--',label=r'$f$')
ax.plot(x,yt,'k--',label=r"$f'$ (exact)")
ax.plot(x,ytf,'b-',label=r"$\hat{f}'$ (forward)")
ax.plot(x,ytb,'g-',label=r"$\hat{f}'$ (backward)")
ax.plot(x,ytc,'m-',label=r"$\hat{f}'$ (central)")

ax.annotate(r'$\frac{f(x_i+h) - f(x_i)}{h}$', xy=(x0, y0), xytext=(x0+h, y0), xycoords='data', textcoords='data',
            arrowprops={'arrowstyle': '<->','color':'b'},color='b')
ax.annotate('', xy=(x0+h, f(x0+h)), xytext=(x0+h, y0), xycoords='data', textcoords='data',
            arrowprops={'arrowstyle': '<->','color':'b'},color='b')

ax.annotate(r'$\frac{f(x_i)-f(x_i-h)}{h}$', xy=(x0-h, f(x0-h)), xytext=(x0, f(x0-h)), xycoords='data', textcoords='data',
            arrowprops={'arrowstyle': '<->','color':'g'},color='g')
ax.annotate('', xy=(x0, y0), xytext=(x0, f(x0-h)), xycoords='data', textcoords='data',
            arrowprops={'arrowstyle': '<->','color':'g'},color='g')

ax.annotate(r'$\frac{f(x_i+h)-f(x_i-h)}{2h}$', xy=(x0+h, f(x0+h)), xytext=(x0-h, f(x0+h)), xycoords='data', textcoords='data',
            arrowprops={'arrowstyle': '<->','color':'m'},color='m',horizontalalignment='right')
ax.annotate('', xy=(x0-h, f(x0-h)), xytext=(x0-h, f(x0+h)), xycoords='data', textcoords='data',
            arrowprops={'arrowstyle': '<->','color':'m'},color='m')

ax.spines['right'].set_visible(False)
ax.spines['top'].set_visible(False)

plt.xticks([x0-h, x0, x0+h], [r'$x_i-h$', r'$x_i$', r'$x_i+h$'])
plt.yticks([f(x0-h), f(x0), f(x0+h)], [r'$f(x_i-h)$', r'$f(x_i)$', r'$f(x_i+h)$'])
plt.title('geometric interpretation (1st order derivative)')
plt.legend(loc='lower right')
plt.show()
```

## Second-order finite difference
From (1)+(2) we obtain:
$$\begin{align*}f''_\text{central}(x) &=\displaystyle \frac{f(x+h) - 2f(x)+f(x-h)}{h^2}
\color{red}{+\frac{h^2}{12}f^{(4)}(x)+\dots}\\
                      &=\displaystyle \frac{f(x+h) - 2f(x) + f(x-h)}{h^2}  \color{red}{+\mathcal{O}(h^2)}
\end{align*}$$

Ref: Chapra, Canale: Numerical Methods for Engineers

Here's an example of a function whose derivatives we know analytically:

```python
f = lambda x: np.log(x)
x0 = 3
df_ex = 1/x0      #f'(x0)
d2f_ex = -1/x0**2 #f''(x0)
```

```python
def f2d_central2(f, x, h):
    """fd_central2 calculates the derivative of f at x
    with Second-order central finite difference.
    """
    return (f(x + h) - 2*f(x) + f(x-h))/(h**2)
```

**Central**: Let's check if the error really decreases by a factor of 4 as we halve $h$, until ultimately roundoff errors spoil the convergence:

```python
h = .1
for m in np.arange(0,10):
    d2f = f2d_central2(f, x0, h)
    err = np.abs(d2f - d2f_ex)
    if m > 0:
        print('h = {:.2E}, err = {:.2E}, fac = {:.2f}'.format(h, err, err_old/err))
    err_old = err
    h = h/2
```

    h = 5.00E-02, err = 1.54E-05, fac = 4.00
    h = 2.50E-02, err = 3.86E-06, fac = 4.00
    h = 1.25E-02, err = 9.65E-07, fac = 4.00
    h = 6.25E-03, err = 2.41E-07, fac = 4.00
    h = 3.13E-03, err = 6.03E-08, fac = 4.00
    h = 1.56E-03, err = 1.52E-08, fac = 3.98
    h = 7.81E-04, err = 3.78E-09, fac = 4.01
    h = 3.91E-04, err = 1.24E-09, fac = 3.06
    h = 1.95E-04, err = 2.69E-09, fac = 0.46

# Numerical solution of ordinary differential equations

An ordinary differential equation (ODE)  is an equation of the form

$$\frac{dx}{dt}=f(y,t)$$

for an unknown function $y:\mathbf{R}\to \mathbf{R}^d$, $t\mapsto y(t)$, and a right-hand side $f:\mathbf{R} \times \mathbf{R}^d \to \mathbf{R}^d$. The initial value problem is completed by specifying an initial condition

$$y(t_0)=y_0$$

Under certain conditions on $f$, a unique solution exists at least in a neighbourhood of $t_0$.

A simple example is the growth of a bacteria colony, whose growth rate is proportional to the size of the population:

$$\frac{dy}{dt}=rt, y(0)=y_0$$

with a constant $r\in\mathbf{R}$, The exact solution is $y(t) = y_0 \, e^{rt}$.

```python
r=0.8
f=lambda t,y:r*y
y0=1000
a=0; b=2
t_ex = np.linspace(a-.03,b+.03,200)
y_ex = lambda t: y0*np.exp(r*t)
```

The simplest numerical method is **Euler's method**, which is based on the Taylor expansion at first order:

$$y(t+h) = y(t) + h y'(t)+ O(h^2) = y(t) + h \, f(t,h) + O(h^2),$$

where for the second equality we have used the ODE to replace the derivative. This step is repeated $n$ times, with $h=(b-a)/n$, starting at $t=a$ and ending up at $t=b$. An implementation can be found in `euler`.

```python
def euler(f,a,b,y0,n):
    """
    EULER solves the ordinary differential equation with right-hand side f and
    initial value y(a) = y0 on the interval [a, b] with n steps of Euler's method
    """
    h = (b - a)/n
    t,y = a,y0
    t_,y_=[],[]
    while t <= b:
        t_.append(t)
        y_.append(y)
        t += h
        y += h * f(t,y)
    return(np.array(t_),np.array(y_))
```

```python
n = 20
[t_euler, y_euler] = euler(f, a, b, y0, n)
```

From the above equation, the local truncation error of a single Euler step is of the order $O(h^2)$. However after the step is repeated $n=(b-a)/h$ times, the global truncation error at $t=b$ is only $O(h)$. So when the number of subintervals is doubled, the error decreases by a factor of 2.

```python
n = 20
for m in np.arange(0,10):
    tm, ym = euler(f, a, b, y0, n)
    err = abs(ym[n-1] - y_ex(b))
    if m > 0:
        print('n = {:}, err = {:.2f}, fac = {:.2f}'.format(n, err, err_old/err))
    err_old = err
    n = n*2
```

    n = 40, err = 336.67, fac = 1.89
    n = 80, err = 173.19, fac = 1.94
    n = 160, err = 87.86, fac = 1.97
    n = 320, err = 44.25, fac = 1.99
    n = 640, err = 22.21, fac = 1.99
    n = 1280, err = 11.12, fac = 2.00
    n = 2560, err = 5.57, fac = 2.00
    n = 5120, err = 2.78, fac = 2.00
    n = 10240, err = 1.39, fac = 2.00

There are more accurate schemes, e.g. the popular 4th-order Runge-Kutta method `rk4`:

```python
def rk4(f,a,b,y0,n):
    """
    RK4 solves the ordinary differential equation with right-hand side f and
    initial value y(a) = y0 on the interval [a, b] with n steps of 4th-order
    Runge-Kutta method
    """
    dy=lambda t, y, dt: (
            lambda dy1: (
            lambda dy2: (
            lambda dy3: (
            lambda dy4: (dy1 + 2*dy2 + 2*dy3 + dy4)/6
            )( dt * f( t + dt  , y + dy3   ) )
            )( dt * f( t + dt/2, y + dy2/2 ) )
            )( dt * f( t + dt/2, y + dy1/2 ) )
            )( dt * f( t       , y         ) )
    h = (b - a)/n
    t,y = a,y0
    t_,y_=[],[]
    while t <= b:
        t_.append(t)
        y_.append(y)
        t += h
        y += dy(t,y,h)
    return(np.array(t_),np.array(y_))

n = 20
[t_rk4, y_rk4] = rk4(f, a, b, y0, n)
```

`scipy` provides `solve_ivp` as interface to various solvers for initial value problems of systems of ODEs.
- Explicit Runge-Kutta methods ('RK23', 'RK45', 'DOP853') should be used for non-stiff problems
- Implicit methods ('Radau', 'BDF') for stiff problems
- Among Runge-Kutta methods, 'DOP853' is recommended for solving with high precision (low values of `rtol` and `atol`)

```python
RK23  =solve_ivp(f,(a, b),[y0],method='RK23')
RK45  =solve_ivp(f,(a, b),[y0],method='RK45')
DOP853=solve_ivp(f,(a, b),[y0],method='DOP853')
Radau =solve_ivp(f,(a, b),[y0],method='Radau')
BDF   =solve_ivp(f,(a, b),[y0],method='BDF')
```

Observe how the step size is adaptive (non-uniform) in this case.

```python
fig, ax = plt.subplots(figsize=(8, 5))
ax.plot(t_ex   ,y_ex(t_ex) , label=r'exact')
ax.plot(t_euler, y_euler, label=r'euler');
ax.plot(t_rk4  , y_rk4  , label=r'rk4');
ax.plot(RK23.t,   RK23.y[0,:],   'o', label=r'RK23');
ax.plot(RK45.t,   RK45.y[0,:],   'o', label=r'RK45');
ax.plot(DOP853.t, DOP853.y[0,:], 'o', label=r'DOP853');
ax.plot(Radau.t,  Radau.y[0,:],  'o', label=r'Radau');
ax.plot(BDF.t,    BDF.y[0,:],    'o', label=r'BDF');
plt.title('Numerical solution of ordinary differential equations')
plt.legend(loc='lower right')
plt.show()
```

```python
t_rk4
```

    array([0. , 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1. , 1.1, 1.2,
           1.3, 1.4, 1.5, 1.6, 1.7, 1.8, 1.9])

## Systems of ODEs, additional parameters

So far we have taken the dimension to be $d=1$. An example of a system with $d=2$ unknown functions is the [Lotka-Volterra (predator-prey) model](https://en.wikipedia.org/wiki/Lotka–Volterra_equations):

$$
\begin{align}
\frac{dy_1}{dt} &= \alpha y_1 - \beta y_1 y_2,\\
\frac{dy_2}{dt} &= \gamma y_1 y_2 - \delta y_2
\end{align}
$$

Here $y_1$ is the population of prey, $y_2$ the population of predators, and $\alpha,\beta,\gamma,\delta>0$ are constants. Each has a certain birthrate and a deathrate. The birthrate of the prey is proportional to its current population and the birthrate of the predator is proportional to both its population and the prey population. The deathrate of the prey is proportional to both its population and the predator population and the deathrate of the predator is proportional to its population.

The equations are non-linear since they include $xy$, this implies that finding an analytical solution will be much harder but for the numerical integration the problem isn't any more difficult.

```python
f = lambda t, y: [p*y[0] - q*y[0]*y[1], r*y[0]*y[1] - s*y[1]]
p = 0.4; q = 0.04; r = 0.02; s = 2
a = 0; b = 15
y0 = [105, 8]
lotka_volterra.main(p, q, r, s, y0[0], y0[1], [a, b])
```

The Population dynamics is shown in the left plot. The population of predators trailing that of prey by 90° in the cycle.

The right plot shows the solutions parametrically as orbits in phase space, without representing time, but with one axis representing the number of prey and the other axis representing the number of predators for all times.

**Population equilibrium** occurs in the model when neither of the population levels is changing, i.e. when both of the derivatives are equal to 0:

$$
\begin{align}
0 &= \alpha y_1 - \beta y_1 y_2,\\
0 &= \gamma y_1 y_2 - \delta y_2
\end{align}
$$

The above system of equations yields two solutions:
$$y_1=0,y_2=0$$
$$y_1=\frac{\delta}{\gamma},y_2=\frac{\alpha}{\beta}$$

The first solution effectively represents the extinction of both species. If both populations are at 0, then they will continue to be so indefinitely. The second solution represents a **fixed point** at which both populations sustain their current, non-zero numbers, and, in the simplified model, do so indefinitely.

```python
print(r'y_1 = {}'.format(s/r))
print(r'y_2 = {}'.format(p/q))
lotka_volterra.main(p, q, r, s, s/r, p/q, [a, b])
```

The **stability of the fixed point** at the origin can be determined by performing a [linearization](https://www.youtube.com/watch?v=k_IkbxwSK7g) using partial derivatives. The linearization of the  Lotka-Volterra system about an equilibrium point $(y_1^*, y_2^*)$ has the form

$$
\begin{bmatrix}
 \frac{du}{dt}\\
 \frac{dv}{dt}
\end{bmatrix}
= \mathbf{J}
\begin{bmatrix}
 u\\
 v
\end{bmatrix}
$$

where $u = y_1 − y_1^*$, $v = y_2 − y_2^*$ and $\mathbf{J}$ is the Jacobian ([Community matrix](https://en.wikipedia.org/wiki/Community_matrix)).

The [Jacobian matrix](https://en.wikipedia.org/wiki/Jacobian_matrix) of the predator–prey model is

$$\mathbf{J}(y_1,y_2)=
\begin{bmatrix}
\alpha - \beta y_2 & -\beta y_1\\
\gamma y_2         & \gamma y_1 - \delta
\end{bmatrix}
$$

The eigenvalues of the Jacobian determine the stability of the equilibrium point. By the [stable manifold theorem](https://en.wikipedia.org/wiki/Stable_manifold_theorem), if one or both eigenvalues of $\mathbf{J}$  have positive real part then the equilibrium is unstable, but if all eigenvalues have negative real part then it is stable.

For the **first fixed point** (extinction) of $(0, 0)$, the Jacobian matrix $\mathbf{J}$ becomes

$$\mathbf{J}(0,0)=
\begin{bmatrix}
\alpha & 0\\
0      & - \delta
\end{bmatrix}
$$

The eigenvalues of this matrix are

$$\lambda_1=\alpha,\lambda_2=-\gamma$$

In the model $\alpha$ and $\gamma$ are always greater than zero, and as such the sign of the eigenvalues above will always differ. Hence the fixed point at the origin is a saddle point.

The stability of this fixed point is of significance. If it were stable, non-zero populations might be attracted towards it, and as such the dynamics of the system might lead towards the extinction of both species for many cases of initial population levels. However, as the fixed point at the origin is a saddle point, and hence unstable, it follows that the extinction of both species is difficult in the model. (In fact, this could only occur if the prey were artificially completely eradicated, causing the predators to die of starvation. If the predators were eradicated, the prey population would grow without bound in this simple model.) The populations of prey and predator can get infinitesimally close to zero and still recover.

For the **second fixed point** of $\left(\frac{\delta}{\gamma},\frac{\alpha}{\beta}\right)$, the Jacobian matrix $\mathbf{J}$ becomes

$$\mathbf{J}\left(\frac{\delta}{\gamma},\frac{\alpha}{\beta}\right)=
\begin{bmatrix}
0 & -\frac{\beta\delta}{\gamma}\\
\frac{\alpha\gamma}{\beta} & 0
\end{bmatrix}
$$

The eigenvalues of this matrix are

$$\lambda_1=i\sqrt{\alpha\delta},\lambda_2=-i\sqrt{\alpha\delta}$$

As the eigenvalues are both purely imaginary and conjugate to each others, this fixed point is elliptic, so the solutions are periodic, oscillating on a small ellipse around the fixed point, with a frequency $\omega=\sqrt{\lambda_1\lambda_2}=\sqrt{\alpha\delta}$ and period $T=\frac{2\pi}{\sqrt{\lambda_1\lambda_2}}$.

As illustrated in the circulating oscillations in the figure above, the level curves are closed orbits surrounding the fixed point.

```python
print('T = {}'.format(2*np.pi/np.sqrt(p*s)))
```

    T = 7.024814731040727

Here the interactive version of this model:

```python
lotka_volterra.interactive()
```

    interactive(children=(FloatSlider(value=0.4, description='Birth Rate of Prey', layout=Layout(width='99%'), max…

**Note**: In real-life situations, prey fluctuations of the discrete numbers of individuals, as well as the family structure and life-cycle of prey, might cause the prey to actually go extinct, and, by consequence, the predators as well.

## Higher-order ODEs

Higher-order ODEs can be rewritten as systems of first-order ODEs. For example, the second-order initial value problem

$$\frac{d^2 y}{dt^2} = -k^2 y, \quad y(0) = y_0, \quad y'(0) = y_0'$$

is equivalent to the following first-order system obtained by introducing the auxiliary variables $y_1 := y$ and y_2 := y':

$$\frac{dy_1}{dt} = y_2, \quad \frac{dy_2}{dt} = -k^2 y_1, \quad y_1(0) = y_0, \quad y_2(0) = y_0'.$$

```python
k = np.pi
f = lambda t, y: [y[1], -k**2*y[0]]
a = 0; b = 2
y0 = [1, 0]
DOP853 = solve_ivp(f, [a,b],y0, method='DOP853')
y_ex = lambda t: np.cos(k*t)
t_ex = np.linspace(a,b,200)

fig, ax = plt.subplots(figsize=(8, 5))
ax.plot(t_ex   ,y_ex(t_ex) , label=r'exact')
ax.plot(DOP853.t, DOP853.y[0,:], 'o', label=r'DOP853');
plt.title('Numerical solution of ordinary differential equations')
plt.legend(loc='lower right')
plt.show()
```

# Simulation of random processes

Recall that `np.random.rand` generates uniformly distributed random numbers between 0 and 1, whereas `np.random.randn` generates normally distributed random numbers with $\mu=0$ and $\sigma=1$.

Tossing a coin can be simulated by dividing the interval  into two equal parts:

```python
r = np.random.rand()
if r > 0.5:
    print('head')
else:
    print('tail')
```

    tail

A nice application of the Monte Carlo method is to estimate $\pi$. Take a square of side length 2 and inscribe a circle of radius 1. Generate uniformly distributed pairs (i.e. points in $\mathbf{R}^2$) of random numbers in $[0,2]^2$. Then the ratio of points falling into the circle to the total number of points should be the ratio of the areas of the circle and the square, namely $\pi/4$. We construct a logical vector to detect and count the points inside the circle.

```python
n = 1000000
x = 2*np.random.rand(2, n)
incircle = (x[0,:] - 1)**2 + (x[1,:] - 1)**2 < 1
pi_estimated = 4*sum(incircle)/n
pi_estimated
```

    3.139724


```python
%pylab inline
```

    Populating the interactive namespace from numpy and matplotlib

```python
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import proj3d, Axes3D
from sympy import symbols, Matrix, solve
import math
from sympy.parsing.sympy_parser import parse_expr
```

```python
plt.figure(figsize=(10,10))
t = symbols('t')
u = parse_expr('Matrix([t*cos(3*t), t*sin(2*t)])')
a = 16
lim = [0, a]
ts = np.linspace(lim[0], lim[1], a**2)
Y = np.array([u.subs('t', t).evalf() for t in ts]).astype('float')
xs = [x[0] for x in Y]
ys = [x[1] for x in Y]
plot(xs, ys)
```

```python
t = symbols('t')
u = parse_expr('Matrix([t*cos(3*t), t*sin(2*t), t])')
a = 16
lim = [0, a]
ts = np.linspace(lim[0], lim[1], a**2)
Y = np.array([u.subs('t', t).evalf() for t in ts]).astype('float')
xs = [x[0] for x in Y]
ys = [x[1] for x in Y]
zs = [x[2] for x in Y]
fig = plt.figure(figsize=(10,10))
#plt.rcParams['savefig.dpi'] = 600
ax = fig.add_subplot(111, projection='3d')
plot(xs, ys, zs, c="purple")
plt.show()
```

```python
type(u)
```

    sympy.matrices.dense.MutableDenseMatrix

```python
u.shape
```

    (3, 1)

```python
def is_matrix_3x1(u): return 'Matrix' in str(type(u)) and u.shape == (3, 1)
def is_matrix_2x1(u): return 'Matrix' in str(type(u)) and u.shape == (2, 1)
```

```python
is_matrix_3x1(u)
```

    True

```python
is_matrix_2x1(u)
```

    False

```python
u.free_symbols
```

    {t}

# 1 var extremas

```python
x = symbols('x')
expr = parse_expr('-x**3 + 10*x**2 - x')
xs = np.linspace(-2, 10, 20)
ys = np.array([expr.subs('x', x).evalf() for x in xs]).astype('float')
plot(xs, ys)
```

```python
expr.diff('x')
```

    -3*x**2 + 20*x - 1

```python
expr_dx = expr.diff(x)
xs = np.linspace(-2, 10, 20)
ys = np.array([expr_dx.subs('x', x).evalf() for x in xs]).astype('float')
plot(xs, ys)
axhline(0, c='black', lw=1, ls='dashed')
cp = solve(expr_dx, 'x')
plot(cp, 'o')
```

```python

```