```python
import pandas as pd
import seaborn as sns
sns.set(context="notebook",style = "whitegrid")
import matplotlib.pyplot as plt
import tensorflow as tf
import numpy as np
```

```python
df = pd.read_csv('ex2data1.txt',names=['exml1','exml2','isadmit'])
df.head()
```

      exml1
      exml2
      isadmit

      0
      34.623660
      78.024693
      0

      1
      30.286711
      43.894998
      0

      2
      35.847409
      72.902198
      0

      3
      60.182599
      86.308552
      1

      4
      79.032736
      75.344376
      1

```python
sns.lmplot(x='exml1',y='exml2',data = df,hue='isadmit',height=6,fit_reg=False)
```

# Sigmoid

 $$
 g(z) = \frac{1}{1+e^{-z}}
 $$

$$
h_{\theta}(x) = \frac{1}{1+e^{-{\theta}^T{X}}}
$$

```python
def sigmoid(z):
    return 1/(1+np.exp(-z))
```

$$
J\left( \theta \right)=\frac{1}{m}\sum\limits_{i=1}^{m}{[-{{y}^{(i)}}\log \left( {{h}_{\theta }}\left( {{x}^{(i)}} \right) \right)-\left( 1-{{y}^{(i)}} \right)\log \left( 1-{{h}_{\theta }}\left( {{x}^{(i)}} \right) \right)]}
$$

```python
df.insert(0,'Ones',1)
```

```python
cols = df.shape[1]
X = np.array(df.iloc[:,0:cols-1].values)
Y = np.array(df.iloc[:,cols-1:cols].values)
theta = np.array(np.zeros(3))
```

```python
X.shape,theta.shape,Y.shape,(X*theta.T).shape
```

    ((100, 3), (3,), (100, 1), (100, 3))

```python
def cost(theta,X,Y):
    theta = np.matrix(theta)
    X = np.matrix(X)
    y = np.matrix(Y)
    first = np.multiply(-Y, np.log(sigmoid(X * theta.T)))
    second = np.multiply((1 - Y), np.log(1 - sigmoid(X * theta.T)))
    return (np.sum(first-second))/(len(X))
```

```python
cost(theta,X,Y)
```

    0.6931471805599453

# Gradient descent

$$\frac{\partial J\left( \theta \right)}{\partial {{\theta }_{j}}}=\frac{1}{m}\sum\limits_{i=1}^{m}{({{h}_{\theta }}\left( {{x}^{(i)}} \right)-{{y}^{(i)}})x_{_{j}}^{(i)}}$$

 $$
 {\theta_j}:={\theta_j}-\alpha \frac{1}{m}\sum\limits_{i=1}^{m}{({h_\theta}({{x}^{(i)}})-{{y}^{(i)}}){x_{j}}^{(i)}}
 $$

```python
def gradient_descent(theta,X,Y):
    theta = np.matrix(theta)
    X = np.matrix(X)
    Y = np.matrix(Y)
    parameters = int(theta.shape[1])
    error = sigmoid(X*theta.T) - Y
    grad = np.zeros(parameters)
    for i in range(parameters):
        term = np.multiply(error,X[:,i])
        grad[i] = np.sum(term)/len(X)
    return grad
```

```python
gradient_descent(theta,X,Y)
```

    array([ -0.1       , -12.00921659, -11.26284221])

```python
import scipy.optimize as opt
result = opt.fmin_tnc(func=cost, x0=theta, fprime=gradient_descent, args=(X, Y))
result
```

    (array([-25.16131872,   0.20623159,   0.20147149]), 36, 0)

```python
cost(result[0],X,Y)
```

    0.20349770158947425

```python
def predict(theta,X):
    probability = sigmoid(X*theta.T)
    return [1 if x>= 0.5 else 0 for x in probability]
```

```python
theta_min = np.matrix(result[0])
predictions = predict(theta_min,X)
correct = [1 if ((a==1 and b==1) or (a == 0 and b == 0)) else 0 for (a,b) in zip(predictions,Y)]
accuracy = (sum(map(int,correct)) % len(correct))
print ('accuracy = {0}%'.format(accuracy))
```

    accuracy = 89%

```python
path =  'ex2data2.txt'
df2 = pd.read_csv(path, header=None, names=['Test 1', 'Test 2', 'Accepted'])
df2.head()
```

      Test 1
      Test 2
      Accepted

      0
      0.051267
      0.69956
      1

      1
      -0.092742
      0.68494
      1

      2
      -0.213710
      0.69225
      1

      3
      -0.375000
      0.50219
      1

      4
      -0.513250
      0.46564
      1

```python
sns.lmplot(x='Test 1',y='Test 2',data = df2,hue='Accepted',height=6,fit_reg=False)
```

```python
#构造多项式
degree = 5
x1 = df2['Test 1']
x2 = df2['Test 2']

df2.insert(3, 'Ones', 1)

for i in range(1, degree):
    for j in range(0, i):
        df2['F' + str(i) + str(j)] = np.power(x1, i-j) * np.power(x2, j)

df2.drop('Test 1', axis=1, inplace=True)
df2.drop('Test 2', axis=1, inplace=True)

df2.head()
```

      Accepted
      Ones
      F10
      F20
      F21
      F30
      F31
      F32
      F40
      F41
      F42
      F43

      0
      1
      1
      0.051267
      0.002628
      0.035864
      0.000135
      0.001839
      0.025089
      0.000007
      0.000094
      0.001286
      0.017551

      1
      1
      1
      -0.092742
      0.008601
      -0.063523
      -0.000798
      0.005891
      -0.043509
      0.000074
      -0.000546
      0.004035
      -0.029801

      2
      1
      1
      -0.213710
      0.045672
      -0.147941
      -0.009761
      0.031616
      -0.102412
      0.002086
      -0.006757
      0.021886
      -0.070895

      3
      1
      1
      -0.375000
      0.140625
      -0.188321
      -0.052734
      0.070620
      -0.094573
      0.019775
      -0.026483
      0.035465
      -0.047494

      4
      1
      1
      -0.513250
      0.263426
      -0.238990
      -0.135203
      0.122661
      -0.111283
      0.069393
      -0.062956
      0.057116
      -0.051818

$$
J\left( \theta \right)=\frac{1}{m}\sum\limits_{i=1}^{m}{[-{{y}^{(i)}}\log \left( {{h}_{\theta }}\left( {{x}^{(i)}} \right) \right)-\left( 1-{{y}^{(i)}} \right)\log \left( 1-{{h}_{\theta }}\left( {{x}^{(i)}} \right) \right)]}+\frac{\lambda }{2m}\sum\limits_{j=1}^{n}{\theta _{j}^{2}}
$$

$$
    \begin{align} & Repeat\text{ }until\text{ }convergence\text{ }\!\!\{\!\!\text{ } \\ & \text{ }{{\theta }_{0}}:={{\theta }_{0}}-a\frac{1}{m}\sum\limits_{i=1}^{m}{[{{h}_{\theta }}\left( {{x}^{(i)}} \right)-{{y}^{(i)}}]x_{_{0}}^{(i)}} \\ & \text{ }{{\theta }_{j}}:={{\theta }_{j}}-a\frac{1}{m}\sum\limits_{i=1}^{m}{[{{h}_{\theta }}\left( {{x}^{(i)}} \right)-{{y}^{(i)}}]x_{j}^{(i)}}+\frac{\lambda }{m}{{\theta }_{j}} \\ & \text{ }\!\!\}\!\!\text{ } \\ & Repeat \\ \end{align}
    $$

 $$
 {{\theta }_{j}}:={{\theta }_{j}}(1-a\frac{\lambda }{m})-a\frac{1}{m}\sum\limits_{i=1}^{m}{({{h}_{\theta }}\left( {{x}^{(i)}} \right)-{{y}^{(i)}})x_{j}^{(i)}}
 $$

```python
def cost2(theta,X,Y,learningRate):
    theta = np.matrix(theta)
    X = np.matrix(X)
    Y = np.matrix(Y)
    first = np.multiply(-Y, np.log(sigmoid(X * theta.T)))
    second = np.multiply((1 - Y), np.log(1 - sigmoid(X * theta.T)))
    reg = (learningRate/(2*len(X)))*np.sum(np.power(theta[:,1:theta.shape[1]],2))
    return np.sum(first-second)/(len(X))+reg
```

```python
def gradientReg_descent(theta,X,Y,learningRate):
    theta = np.matrix(theta)
    X = np.matrix(X)
    Y = np.matrix(Y)
    parameters = int(theta.shape[1])
    error = sigmoid(X*theta.T) - Y
    grad = np.zeros(parameters)
    for i in range(parameters):
        term = np.multiply(error,X[:,i])
        if i == 0:
            grad[i] = np.sum(term)/len(X)
        else:
            grad[i] = np.sum(term)/len(X) + ((learningRate/len(X)))*theta[:,i]
    return grad
```

```python
cols = df2.shape[1]
X2 = np.array(df2.iloc[:,1:cols].values)
Y2 = np.array(df2.iloc[:,0:1].values)
theta2 = np.array(np.zeros(11))
```

```python
learningRate = 1
```

```python
cost2(theta2,X2,Y2,learningRate)
```

    0.6931471805599454

```python
gradientReg_descent(theta2,X2,Y2,learningRate)
```

    array([0.00847458, 0.01878809, 0.05034464, 0.01150133, 0.01835599,
           0.00732393, 0.00819244, 0.03934862, 0.00223924, 0.01286005,
           0.00309594])

```python
result2 = opt.fmin_tnc(func=cost2,x0=theta2,fprime=gradientReg_descent,args=(X2,Y2,learningRate))
result2
```

    (array([ 0.53010248,  0.29075567, -1.60725764, -0.5821382 ,  0.01781027,
            -0.21329508, -0.40024142, -1.37144139,  0.02264303, -0.9503358 ,
             0.0344085 ]), 22, 1)

```python
theta_min = np.matrix(result2[0])
predictions = predict(theta_min, X2)
correct = [1 if ((a == 1 and b == 1) or (a == 0 and b == 0)) else 0 for (a, b) in zip(predictions, Y2)]
accuracy = (sum(map(int, correct)) % len(correct))
print ('accuracy = {0}%'.format(accuracy))
```

    accuracy = 78%

```python
from sklearn import linear_model#调用sklearn的线性回归包
model = linear_model.LogisticRegression(penalty='l2', C=1.0)
model.fit(X2, Y2.ravel())
model.score(X2, Y2)
```

    0.6610169491525424

```python

```


# 07 Solution: Sine Series

* *Computational Physics*:  2.5

## Problem: Summing Series: sin(x)

Evaluate the $\sin$ function from its series representation
$$
\sin x = x - \frac{x^3}{3!} + \frac{x^5}{5!} - \frac{x^7}{7!} + \dots
$$

A naive algorithm is to sum the series up to the $N$th term:
$$
\sin x \approx \sum_{n=1}^N \frac{(-1)^{n-1} x^{2n-1}}{(2n - 1)!}
$$

Problems:

- How to decide when to stop summing?
- Division of large terms (overflows!)
- Powers and factorials are very expensive to compute.

Better approach: Build up series terms $a_n$ using previous term $a_{n-1}$ through a recursion:

\begin{align}
a_n &= a_{n-1} \times q_n\\
a_n &= \frac{(-1)^{n-1} x^{2n-1}}{(2n - 1)!} = \frac{(-1)^{n-2} x^{2n-3}}{(2n - 3)!} \frac{-x^2}{(2n - 1)(2n - 2)}\\
a_n & = a_{n-1} \frac{-x^2}{(2n - 1)(2n - 2)}
\end{align}

Accuracy of this approach? Not clear in absolute terms but we can make the assumption that the error is approximately the last term in the sum, $a_N$. Hence we can strive to make the relative error smaller than the machine precision
$$
\left| \frac{a_N}{\sum_{n=1}^N a_n} \right| < \epsilon_m
$$

```python
import numpy as np
```

```python
def sin_series(x, eps=1e-16):
    """Calculate sin(x) to precision eps with recursion.

    Arguments
    ---------
    x : float
        argument of sin(x)
    eps : float, optional
        desired precision

    Returns
    -------
    func_value, N

    where func_value = sin(x) and N is the number
    of terms included in the approximation.
    """
    if x == 0:
        return 0., 1
    a_n = sumN = x   # a_1 is the sum
    n = 2            # start with calculating a_2
    while abs(a_n/sumN) > eps:
        a_n *= -x*x/((2*n - 1)*(2*n - 2))
        sumN += a_n
        n += 1
    return sumN, n-1
```

Bonus problem: the really bad implementation:

```python
import math
def sin_series_bad(x, eps=1e-16):
    """Calculate sin(x) to precision eps... BADLY"""
    if x == 0:
        return 0., 1
    a_n = x
    n = 1
    sumN = 0.
    while True:
        a_n = (-1)**(n-1) * x**(2*n-1) / math.factorial(2*n-1)
        sumN += a_n
        if abs(a_n/sumN) <=  eps:
            break
        n += 1
    return sumN, n
```

```python
sin_series_bad(14*np.pi)
```

    (253.21339842367362, 73)

```python
sin_series(100*np.pi, eps=1e-16)
```

    (6.591899418779782e+118, 277)

Test the implementation against the "exact" numpy function `np.sin()`.

Report
1. `x`
2. maximum `n`
3. `sin_series(x)`
4. relative error `abs(sin_series(x) - sin(x))/abs(sin(x))`

Plot against `x` the quantities above and also `sin(x)`.
* `x` - `sin_series(x)`
* `x` - `sin(x)`
* `x` - max `n`
* `x` - relative error (semilogy plot!)

For a range of numbers look at `Xsmall` and `Xlarge`:

```python
Xsmall = np.pi*np.arange(-2, 2.05, 0.05)
Xlarge = np.pi*np.arange(-50, 50.05, 0.1)
```

Implementation (modularized compared to the problem so that I can use it for `sin_series()` and `sin_series_bad()`):

```python
import numpy as np
def test_sin(x):
    y, nmax = sin_series(x)
    return compare_to_sin(x, y, nmax)

def test_sin_bad(x):
    y, nmax = sin_series_bad(x)
    return compare_to_sin(x, y, nmax)

def compare_to_sin(x, y, nmax):
    y0 = np.sin(x)
    delta = y - y0
    if y0 != 0:
        relative_error = delta/y0
    else:
        relative_error = 0
    # print(x, y, y0, delta, relative_error)
    #return x, y, y0, delta, relative_error
    return x, nmax, y, relative_error, delta
```

The functions do not work as numpy ufuncs on whole arrays; therefore, loop over all input values $x$ like the following code:

```python
results = np.array([test_sin(x) for x in Xsmall])
```

Plot the approximation and the exact solution:

```python
import matplotlib.pyplot as plt
%matplotlib inline
```

```python
def test_plot_sine(X, filename="sine_error.pdf"):
    results = np.array([test_sin(x) for x in X])
    return plot_results(results, filename)

def test_plot_bad_sine(X, filename="sine_bad_error.pdf"):
    results = np.array([test_sin_bad(x) for x in X])
    return plot_results(results, filename)

def plot_results(results, filename):
    fig = plt.figure(figsize=(6, 8))

    ax1 = fig.add_subplot(4,1,1)
    ax1.plot(results[:, 0], results[:, 2], 'k-', lw=1, label="series")
    ax1.plot(results[:, 0], np.sin(results[:, 0]), 'g--', lw=2, label="sin x")
    ax1.legend(loc="best")

    ax2 = fig.add_subplot(4,1,2)
    ax2.plot(results[:, 0], results[:, 1], label="max n")
    ax2.legend(loc="best")

    ax3 = fig.add_subplot(4,1,3)
    ax3.semilogy(results[:, 0], results[:, 3], label="rel.error")
    ax3.legend(loc="best")

    ax4 = fig.add_subplot(4, 1, 4)
    ax4.semilogy(results[:, 0], results[:, 4], label="abs.error")
    ax4.legend(loc="best")

    fig.suptitle("sine series approximation")

    fig.savefig(filename)
    print("saved to file {0}".format(filename))
```

Now test the two ranges of numbers and write to different files:

For a few oscillations near $x=0$ the approximation appears to hold pretty well with relative errors near machine precision.

```python
test_plot_sine(Xsmall, filename="sine_error_Xsmall.pdf")
```

But for larger $|x|$, the approximation breaks down and round-off error completely swamps the result:

```python
test_plot_sine(Xlarge, filename="sine_error_Xlarge.pdf")
```

#### More exploration

```python
test_plot_sine(np.pi*np.arange(-5, 5, 0.05))
```

```python
test_plot_sine(np.pi*np.arange(-10, 10, 0.05))
```

```python
test_plot_sine(np.pi*np.arange(-50, 50, 0.1))
```

```python

```


```python
import os
import sys
import glob
import operator as op
import itertools as it
from functools import reduce, partial
import numpy as np
import pandas as pd
from pandas import DataFrame, Series
import matplotlib.pyplot as plt
import seaborn as sns
sns.set_context("notebook", font_scale=1.5)
%matplotlib inline

from sympy import symbols, hessian, Function, N
```

# Algorithms for Optimization and Root Finding for Multivariate Problems

## Optimization/Roots in n Dimensions - First Some Calculus

Let's review the theory of optimization for multivariate functions. Recall that in the single-variable case, extreme values (local extrema) occur at points where the first derivative is zero, however, the vanishing of the first derivative is not a sufficient condition for a local max or min.  Generally, we apply the second derivative test to determine whether a candidate point is a max or min (sometimes it fails - if the second derivative either does not exist or is zero).  In the multivariate case, the first and second derivatives are *matrices*.  In the case of a scalar-valued function on $\mathbb{R}^n$, the first derivative is an $n\times 1$ vector called the *gradient* (denoted $\nabla f$). The second derivative is an $n\times n$ matrix called the *Hessian* (denoted $H$)

Just to remind you, the gradient and Hessian are given by:

$$\nabla f(x) = \left(\begin{matrix}\frac{\partial f}{\partial x_1}\\ \vdots \\\frac{\partial f}{\partial x_n}\end{matrix}\right)$$

$$H = \left(\begin{matrix}
  \dfrac{\partial^2 f}{\partial x_1^2} & \dfrac{\partial^2 f}{\partial x_1\,\partial x_2} & \cdots & \dfrac{\partial^2 f}{\partial x_1\,\partial x_n} \\[2.2ex]
  \dfrac{\partial^2 f}{\partial x_2\,\partial x_1} & \dfrac{\partial^2 f}{\partial x_2^2} & \cdots & \dfrac{\partial^2 f}{\partial x_2\,\partial x_n} \\[2.2ex]
  \vdots & \vdots & \ddots & \vdots \\[2.2ex]
  \dfrac{\partial^2 f}{\partial x_n\,\partial x_1} & \dfrac{\partial^2 f}{\partial x_n\,\partial x_2} & \cdots & \dfrac{\partial^2 f}{\partial x_n^2}
\end{matrix}\right)$$

One of the first things to note about the Hessian - it's symmetric. This structure leads to some useful properties in terms of interpreting critical points.

The multivariate analog of the test for a local max or min turns out to be a statement about the gradient and the Hessian matrix.  Specifically, a function $f:\mathbb{R}^n\rightarrow \mathbb{R}$ has a critical point at $x$ if $\nabla f(x) = 0$ (where zero is the zero vector!).  Furthermore, the second derivative test at a critical point is as follows:

* If $H(x)$ is positive-definite ($\iff$ it has all positive eigenvalues), $f$ has a local minimum at $x$
* If $H(x)$ is negative-definite ($\iff$ it has all negative eigenvalues), $f$ has a local maximum at $x$
* If $H(x)$ has both positive and negative eigenvalues, $f$ has a saddle point at $x$.

### Note: much of the following notes are taken from Nodecal, J., and S. J. Wright. "Numerical optimization." (2006). It is available online via the library.

## Convexity

A subset $A\subset \mathbb{R}^n$ is *convex* if for any two points $x,y\in A$, the line segment:

$$tx + (1-t)y \;\;\;\;\;\; t\in [0,1]$$
is also in $A$

A function $f:\mathbb{R}^n \rightarrow \mathbb{R}$ is *convex* if its domain $D$ is a convex set and for any two points $x,y\in D$, the graph of $f$ (a subset of $\mathbb{R}^{n+1})$ lies below the line:

$$tf(x) + (1-t)f(y)\;\;\;\;\;t\in [0,1]$$

i.e.
$$f(tx+(1-t)y) \leq tf(x) + (1-t)f(y)\;\;\;\;\;t\in [0,1]$$

### Convexity guarantees that if an optimizer converges, it converges to the global minimum.

Luckily, we often encounter convex problems in statistics.

## Line Search Methods

There are essentially two classes of multivariate optimization methods. We'll cover line search methods, but refer the reader to Nodecal and Wright for discussion of 'trust region methods'. We should note that all of these methods require that we are 'close' to the minimum (maximum) we are seeking, and that 'noisy' functions or ill-behaved functions are beyond our scope.

A line search method is exactly as it sounds - we search on a line (in $n$ dimensional space) and try to find a minimum.  We start with an initial point, and use an iterative method:

$$x_{k+1} = x_k + \alpha_k p_k$$

where $\alpha_k$ is the *step size* and $p_k$ is the search direction. These are the critical choices that change the behavior of the search.

### Step Size

Ideally, (given a choice of direction, $p_k$) we would want to minimize:

$$\varphi(\alpha) = f(x_k + \alpha p_k)$$

with respect to $\alpha$. This is usually computationally intensive, so in practice, a sequence of $\alpha$ candidates are generated, and then the 'best' is chosen according to some 'conditions'. We won't be going into detail regarding these. The important thing to know is that they ensure that $f$ decreases sufficiently, according to some conditions. Interested students should see Nodecal.

## Steepest Descent

In steepest descent, one chooses $p_k=\nabla f_k = \nabla f(x_k)$. It is so named, because the gradient points in the direction of steepest ascent, thus, $-\nabla f_k$ will point in the direction of steepest descent. We'll consider this method in its ideal case, that of a quadratic:

$$f(x) = \frac12 x^TQx - b^Tx$$

where $Q$ is positive-definite and symmetric. Note that:

$$\nabla f = Qx -b$$

so the minimum occurs at $x$ such that

$$Qx= b$$

Clearly, we can solve this easily, but let's walk through the algorithm and first find the (ideal) step length:

$$f(x_k - \alpha \nabla f_k) = \frac12\left(x_k - \alpha \nabla f_k\right)^TQ\left(x_k - \alpha \nabla f_k\right) - b^T \left(x_k - \alpha \nabla f_k\right) $$

If we differentiate this with respect to $\alpha$ and find the zero, we obtain:

$$\alpha_k = \frac{\nabla f_k^T\nabla f_k}{\nabla f_k^TQ\nabla f_k}$$

Thus,

$$x_{k+1} = x_k - \frac{\nabla f_k^T\nabla f_k}{\nabla f_k^TQ\nabla f_k} \nabla f_k$$

But we know that $\nabla f_k = Qx_k -b$, so we have a closed form solution for $x_{k+1}$. This allows us to compute an error bound. Again, details can be found in the text, but here is the result:

$$||x_{k+1} - x^*||_Q^2 \leq \left(\frac{\lambda_n - \lambda_1}{\lambda_n+\lambda_1}\right)^2 ||x_{k} - x^*||_Q^2$$

where $0<\lambda_1\leq ... \leq \lambda_n$ and $x^*$ denotes the minimizer.

Now, if $\lambda_1=...=\lambda_n = \lambda$, then $Q=\lambda I$, the algorithm converges in one step. Geometrically, the contours are ellipsoids, the value of $\frac{\lambda_n}{\lambda_1}$ elongates the axes and causes the steps to 'zig-zag'. Because of this, convergence slows as $\frac{\lambda_n}{\lambda_1}$ increases.

## Newton's Method

Newton's method is another line-search, and here

$$p_k = -H^{-1}\nabla f_k$$

Note that if the Hessian is not positive definite, this may not always be a descent direction.

In the neighborhood of a local minimum, the Hessian *will* be positive definite. Now, if $x_0$ is 'close enough' to the minimizer $x^*$, the step size $\alpha_k =1$ gives quadratic convergence.

The advantage of multiplying the gradient by the inverse of the Hessian is that the gradient is corrected for curvature, and the new direction points toward the minimum.

```python

#def Quad(x):
#    return (x[1:])*np.sin(x[:-1])**2.0)

#def DQuad(x,y):
#    return (np.array([np.cos(x)*np.sin(y)**2.0,2.0*np.sin(x)*np.cos(y)**2.0]))

def Quad(x):
    return ((x[1:])**2.0 + 5*(x[:-1])**2.0)

def DQuad(x,y):
    return (np.array([2.0*x,10.0*y]))
```

```python
x = np.linspace(-20,20, 100)
y = np.linspace(-20,20, 100)
X, Y = np.meshgrid(x, y)
Z = Quad(np.vstack([X.ravel(), Y.ravel()])).reshape((100,100))
Hinv=-np.array([[0.5,0],[0,0.1]])
```

```python
plt.figure(figsize=(12,4))
plt.subplot(121)

plt.contour(X,Y,Z);
plt.title("Steepest Descent");
step=-0.25
X0 = 10.0
Y0 = 1.0

Ngrad=Hinv.dot(DQuad(X0,Y0))

sgrad = step*DQuad(X0,Y0)
plt.quiver(X0,Y0,sgrad[0],sgrad[1],color='red',angles='xy',scale_units='xy',scale=1);
X1 = X0 + sgrad[0]
Y1 = Y0 + sgrad[1]
sgrad = step*DQuad(X1,Y1)
plt.quiver(X1,Y1,sgrad[0],sgrad[1],color='green',angles='xy',scale_units='xy',scale=1);
X2 = X1 + sgrad[0]
Y2 = Y1 + sgrad[1]
sgrad = step*DQuad(X2,Y2)
plt.quiver(X2,Y2,sgrad[0],sgrad[1],color='purple',angles='xy',scale_units='xy',scale=1);

plt.subplot(122)
plt.contour(X,Y,Z);
plt.title("Newton's Method")
plt.quiver(X0,Y0,Ngrad[0],Ngrad[1],color='purple',angles='xy',scale_units='xy',scale=1);

#Compute Hessian and plot again.

```

## Coordinate Descent

Another method is called 'coordinate' descent, and it involves searching along coordinate directions (cyclically), i.e.:

$$p_{mk} = e_{k} \;\;\;\;\;\; k=1,...,n$$

where $m$ is the number of steps.

The main advantage is that $\nabla f$ is not required. It can behave reasonably well, if coordinates are not tightly coupled.

### Newton CG Algorithm

Features:

* Minimizes a 'true' quadratic on $\mathbb{R}^n$ in $n$ steps
* Does NOT require storage or inversion of an $n \times n$ matrix.

We begin with $:\mathbb{R}^n\rightarrow \mathbb{R}$.  Take a quadratic approximation to $f$:

$$f(x) \approx \frac12 x^T H x + b^Tx + c$$

Note that in the neighborhood of a minimum, $H$ will be positive-definite (and symmetric). (If we are maximizing, just consider $-H$).

This reduces the optimization problem to finding the zeros of

$$Hx = -b$$

This is a linear problem, which is nice.  The dimension $n$ may be very large - which is not so nice.

#### General Inner Product

Recall the axiomatic definition of an inner product $<,>_A$:

* For any two vectors $v,w$ we have
$$<v,w>_A = <w,v>_A$$
* For any vector $v$
$$<v,v>_A \;\geq 0$$
   with equality $\iff$ $v=0$.
* For $c\in\mathbb{R}$ and $u,v,w\in\mathbb{R}^n$, we have
$$<cv+w,u> = c<v,u> + <w,u>$$

These properties are known as symmetric, positive definite and bilinear, respectively.

Fact: If we denote the standard inner product on $\mathbb{R}^n$ as $<,>$ (this is the 'dot product'), any symmetric, positive definite $n\times n$ matrix $A$ defines an inner product on $\mathbb{R}^n$ via:

$$<v,w>_A \; = <v,Aw> = v^TAw$$

Just as with the standard inner product, general inner products define for us a notion of 'orthogonality'. Recall that with respect to the standard product, 2 vectors are orthogonal if their product vanishes. The same applies to $<,>_A$:

$$<v,w>_A = 0 $$

means that $v$ and $w$ are orthogonal under the inner product induced by $A$.  Equivalently, if $v,w$ are orthogonal under $A$, we have:

$$v^TAw = 0$$

This is also called *conjugate* (thus the name of the method).

#### Conjugate Vectors

Suppose we have a set of $n$ vectors $p_1,...,p_n$ that are mutually conjugate.  These vectors form a basis of $\mathbb{R}^n$.  Getting back to the problem at hand, this means that our solution vector $x$ to the linear problem may be written as follows:

$$x = \sum\limits_{i=1}^n \alpha_i p_i$$

So, finding $x$ reduces to finding a conjugate basis and the coefficients for $x$ in that basis.

If we let $A=H$,note that:

$${p}_k^{T} {-b}={p}_k^{T} {A}{x}$$

and because $x = \sum\limits_{i=1}^n \alpha_i p_i$, we have:
$$p^TAx = \sum\limits_{i=1}^n \alpha_i p^TA p_i$$

we can solve for $\alpha_k$:

$$\alpha_k = \frac{{p}_k^{T}{(-b)}}{{p}_k^{T} {A}{p}_k} = -\frac{\langle {p}_k, {b}\rangle}{\,\,\,\langle {p}_k,  {p}_k\rangle_{A}} = -\frac{\langle{p}_k, {b}\rangle}{\,\,\,\|{p}_k\|_{A}^2}.$$

Now, all we need are the $p_k$'s.

A nice initial guess would be the gradient at some initial point $x_1$.  So, we set $p_1 = \nabla f(x_1)$. Then set:

$$x_2 = x_1 + \alpha_1p_1$$

This should look familiar.  In fact, it is gradient descent.  For $p_2$, we want $p_1$ and $p_2$ to be conjugate (under $A$).  That just means orthogonal under the inner product induced by $A$.  We set

$$p_2 = \nabla f(x_2) - \frac{p_1^TA\nabla f(x_2)}{{p}_1^{T}{A}{p}_1} {p}_1$$

I.e. We take the gradient at $x_1$ and subtract its projection onto $p_1$.  This is the same as Gram-Schmidt orthogonalization.

The $k^{th}$ conjugate vector is:

$$p_{k} = \nabla f(x_k) - \sum\limits_{i=1}^{k-1}\frac{p_i^T A \nabla f(x_k)}{p_i^TAp_i} p_i$$

The 'trick' is that in general, we do not need all $n$ conjugate vectors. In fact, it turns out that $\nabla f(x_k) = b-Ax_k$ is conjugate to all the $p_i$ for $i=1,...,k-2$. Therefore, we need only the last term in the sum.

Convergence rate is dependent on sparsity and condition number of $A$.  Worst case is $n^2$.

### BFGS - Broyden–Fletcher–Goldfarb–Shanno

BFGS is a 'quasi' Newton method of optimization.  Such methods are variants of the Newton method, where the Hessian $H$ is replaced by some approximation.  We we wish to solve the equation:

$$B_k{p}_k = -\nabla f({x}_k)$$

for $p_k$. This gives our search direction, and the next candidate point is given by:

$$x_{k+1} = x_k + \alpha_k p_k$$.

where $\alpha_k$ is a step size.

At each step, we require that the new approximate $H$ meets the secant condition:

$$B_{k+1}(x_{k+1}-x_k) = \nabla f(x_{k+1}) -\nabla f(x_k)$$

There is a unique, rank one update that satisfies the above:

$$B_{k+1} = B_k + c_k v_kv_k^T$$

where

$$ c_k = -\frac{1}{\left(B_k(x_{k+1}-x_k) - (\nabla f(x_{k+1})-\nabla f(x_k)\right)^T (x_{k+1}-x_k) }$$

and

$$v_k = B_k(x_{k+1}-x_k) - (\nabla f(x_{k+1})-\nabla f(x_k))$$

Note that the update does NOT preserve positive definiteness if $c_k<0$.  In this case, there are several options for the rank one correction, but we will not address them here.  Instead, we will describe the BFGS method, which almost always guarantees a positive-definite correction.  Specifically:

$$B_{k+1} = B_k + b_k g_k g_k^T + c_k B_k d_k d_k^TB_k$$

where we have introduced the shorthand:

$$g_k = \nabla f(x_{k+1}) - \nabla f(x_k) \;\;\;\;\;\;\;\ \mathrm{ and }\;\;\;\;\;\;\; d_k = x_{k+1} - x_k$$

If we set:

$$b_k = \frac{1}{g_k^Td_k} \;\;\;\;\; \mathrm{ and } \;\;\;\;\; c_k = \frac{1}{d_k^TB_kd_k}$$

we satisfy the secant condition.

### Nelder-Mead Simplex

While Newton's method is considered a 'second order method' (requires the second derivative), and quasi-Newton methods are first order (require only first derivatives), Nelder-Mead is a zero-order method.  I.e. NM requires only the function itself - no derivatives.

For $f:\mathbb{R}^n\rightarrow \mathbb{R}$, the algorithm computes the values of the function on a simplex of dimension $n$, constructed from $n+1$ vertices.  For a univariate function, the simplex is a line segment.  In two dimensions, the simplex is a triangle, in 3D, a tetrahedral solid, and so on.

The algorithm begins with $n+1$ starting points and then the follwing steps are repeated until convergence:

* Compute the function at each of the points
* Sort the function values so that
$$f(x_1)\leq ...\leq f(x_{n+1})$$
* Compute the centroid $x_c$ of the n-dimensional region defined by $x_1,...,x_n$
* Reflect $x_{n+1}$ about the centroid to get $x_r$
$$x_r = x_c + \alpha (x_c - x_{n+1})$$
* Create a new simplex according to the following rules:
    - If $f(x_1)\leq f(x_r) < f(x_n)$, replace $x_{n+1}$ with $x_r$
    - If $f(x_r)<f(x_1)$, expand the simplex through $x_r$:
    $$x_e = x_c + \gamma (x_c - x_{n+1})$$
      If $f(x_e)<f(x_r)$, replace $x_{n+1}$ with $x_e$, otherwise, replace $x_{n+1}$ with $x_r$
    - If $f({x}_{r}) \geq f({x}_{n})$, compute $x_p = x_c + \rho(x_c - x_{n+1})$.  If $f({x}_{p}) < f({x}_{n+1})$, replace $x_{n+1}$ with $x_p$
    - If all else fails, replace *all* points except $x_1$ according to
    $$x_i = {x}_{1} + \sigma({x}_{i} - {x}_{1})$$

The default values of $\alpha, \gamma,\rho$ and $\sigma$ in scipy are not listed in the documentation, nor are they inputs to the function.

### Powell's Method

Powell's method is another derivative-free optimization method that is similar to conjugate-gradient.  The algorithm steps are as follows:

Begin with a point $p_0$ (an initial guess) and a set of vectors $\xi_1,...,\xi_n$, initially the standard basis of $\mathbb{R}^n$.

- Compute for $i=1,...,n$, find $\lambda_i$ that minimizes $f(p_{i-1} +\lambda_i \xi_i)$ and set $p_i = p_{i-1} + \lambda_i\xi_i$
- For $i=1,...,n-1$, replace $\xi_{i}$ with $\xi_{i+1}$ and then replace $\xi_n$ with $p_n - p_0$
- Choose $\lambda$ so that $f(p_0 + \lambda(p_n-p_0)$ is minimum and replace $p_0$ with $p_0 + \lambda(p_n-p_0)$

Essentially, the algorithm performs line searches and tries to find fruitful directions to search.

## Solvers

### Levenberg-Marquardt (Damped Least Squares)

Recall the least squares problem:

Given a set of data points $(x_i, y_i)$ where $x_i$'s are independent variables (in $\mathbb{R}^n$ and the $y_i$'s are response variables (in $\mathbb{R}$), find the parameter values of $\beta$ for the model $f(x;\beta)$ so that

$$S(\beta) = \sum\limits_{i=1}^m \left(y_i - f(x_i;\beta)\right)^2$$
is minimized.

If we were to use Newton's method, our update step would look like:

$$\beta_{k+1} = \beta_k - H^{-1}\nabla S(\beta_k)$$

Gradient descent, on the other hand, would yield:

$$\beta_{k+1} = \beta_k - \gamma\nabla S(\beta_k)$$

Levenberg-Marquardt adaptively switches between Newton's method and gradient descent.

$$\beta_{k+1} = \beta_k - (H + \lambda I)^{-1}\nabla S(\beta_k)$$

When $\lambda$ is small, the update is essentially Newton-Gauss, while for $\lambda$ large, the update is gradient descent.

### Newton-Krylov

The notion of a Krylov space comes from the Cayley-Hamilton theorem (CH).  CH states that a matrix $A$ satisfies its characteristic polynomial.  A direct corollary is that $A^{-1}$ may be written as a linear combination of powers of the matrix (where the highest power is $n-1$).

The Krylov space of order $r$ generated by an $n\times n$ matrix $A$ and an $n$-dimensional vector $b$ is given by:

$$\mathcal{K}_r(A,b) = \operatorname{span} \, \{ b, Ab, A^2b, \ldots, A^{r-1}b \}$$

These are actually the subspaces spanned by the conjugate vectors we mentioned in Newton-CG, so, technically speaking, Newton-CG is a Krylov method.

Now, the scipy.optimize newton-krylov solver is what is known as a 'Jacobian Free Newton Krylov'.  It is a very efficient algorithm for solving *large* $n\times n$ non-linear systems.  We won't go into detail of the algorithm's steps, as this is really more applicable to problems in physics and non-linear dynamics.

## GLM Estimation and IRLS

Recall generalized linear models are models with the following components:

* A linear predictor $\eta = X\beta$
* A response variable with distribution in the exponential family
* An invertible 'link' function $g$ such that
$$E(Y) = \mu = g^{-1}(\eta)$$

We may write the log-likelihood:

$$\ell(\eta) = \sum\limits_{i=1}^m (y_i \log(\eta_i) + (\eta_i - y_i)\log(1-\eta_i) $$

where $\eta_i = \eta(x_i,\beta)$.

Differentiating, we obtain:

$$\frac{\partial L}{\partial \beta} = \frac{\partial \eta}{\partial \beta}^T\frac{\partial L}{\partial \eta} = 0$$

Written slightly differently than we have in the previous sections, the Newton update to find $\beta$ would be:

$$-\frac{\partial^2 L}{\partial \beta \beta^T} \left(\beta_{k+1} -\beta_k\right) = \frac{\partial \eta}{\partial \beta}^T\frac{\partial L}{\partial \eta}$$

Now, if we compute:

$$-\frac{\partial^2 L}{\partial \beta \beta^T} = \sum \frac{\partial L}{\partial \eta_i}\frac{\partial^2 \eta_i}{\partial \beta \beta^T} - \frac{\partial \eta}{\partial \beta}^T \frac{\partial^2 L}{\partial \eta \eta^T}  \frac{\partial \eta}{\partial \beta}$$

Taking expected values on the right hand side and noting:

$$E\left(\frac{\partial L}{\partial \eta_i} \right) = 0$$

and

$$E\left(-\frac{\partial^2 L}{\partial \eta \eta^T} \right) = E\left(\frac{\partial L}{\partial \eta}\frac{\partial L}{\partial \eta}^T\right) \equiv A$$

So if we replace the Hessian in Newton's method with its expected value, we obtain:

$$\frac{\partial \eta}{\partial \beta}^TA\frac{\partial \eta}{\partial \beta}\left(\beta_{k+1} -\beta_k\right) = \frac{\partial \eta}{\partial \beta}^T\frac{\partial L}{\partial \eta} $$

Now, these actually have the form of the normal equations for a weighted least squares problem.

$$\min_{\beta_{k+1}}\left(A^{-1}\frac{\partial L}{\partial \eta} + \frac{\partial \eta}{\partial \beta}\left(\beta_{k+1} -\beta_k\right)\right)^T A \left(A^{-1}\frac{\partial L}{\partial \eta} + \frac{\partial \eta}{\partial \beta}\left(\beta_{k+1} -\beta_k\right)\right)$$

$A$ is a weight matrix, and changes with iteration - thus this technique is *iteratively reweighted least squares*.

### Constrained Optimization and Lagrange Multipliers

Often, we want to optimize a function subject to a constraint or multiple constraints.  The most common analytical technique for this is called 'Lagrange multipliers'.  The theory is based on the following:

If we wish to optimize a function $f(x,y)$ subject to the constraint $g(x,y)=c$, we are really looking for points at which the gradient of $f$ and the gradient of $g$ are in the same direction.  This amounts to:

$$\nabla_{(x,y)}f = \lambda \nabla_{(x,y)}g$$

(often, this is written with a (-) sign in front of $\lambda$). The 2-d problem above defines two equations in three unknowns.  The original constraint, $g(x,y)=c$ yields a third equation.  Additional constraints are handled by finding:

$$\nabla_{(x,y)}f = \lambda_1 \nabla_{(x,y)}g_1 + ... + \lambda_k \nabla_{(x,y)}g_k$$

The generalization to functions on $\mathbb{R}^n$ is also trivial:

$$\nabla_{x}f = \lambda \nabla_{x}g$$

```python

```


# Circular Restricted Three Body Problem Derrivation
*SymPy* is usefull for:
1) Derriving the full equations of motion
2) Optimizing code through common subexpression elimination
3) Generating complicated $\LaTeX$ expressions. *SymPy* is **open-source**.

```python
from sympy import *
```

```python
# State
x, y, z, vx, vy, vz, m = symbols('x y z vx vy vz m')
r                      = Matrix([x, y, z])
v                      = Matrix([vx, vy, vz])
s                      = Matrix([r, v, [m]])

# Costate
lx, ly, lz, lvx, lvy, lvz, lm = symbols('lx ly lz lvx lvy lvz lm')
lr                            = Matrix([lx, ly, lz])
lv                            = Matrix([lvx, lvy, lvz])
l                             = Matrix([lr, lv, [lm]])

# Full State
fs = Matrix([s, l])

# Controls
u, ax, ay, az = symbols('u ax ay az')
a             = Matrix([ax, ay, az])
cont          = Matrix([[u], a])

# Parameters
Isp, g0, mu, T, eps = symbols('Isp g0 mu T eps')
c                   = Isp*g0
```

```python
# Intermediates
r1 = sqrt((x + mu)**2 + y**2 + z**2)
r2 = sqrt((x + mu -1)**2 + y**2 + z**2)
```

```python
# Position Dependence
g = Matrix([
        x - (1 - mu)*(x + mu)/r1**3 - mu*(x + mu - 1)/r2**3,
        y - (1 - mu)*y/r1**3 - mu*y/r2**3,
        -(1 - mu)*z/r1**3 - mu*z/r2**3
    ])
# Velocity Dependence
h = Matrix([2*vy, -2*vx, 0])
```

```python
# State Equations of Motion
ds = Matrix([v, g + h + u*T*a/m, [-u*T/c]])
```

```python
# Homotopic Cost Lagrangian
L = T*(u - eps*u*(1 - u))/c
```

```python
# Hamiltonian
Hamiltonian = l.dot(ds) + L
```

```python
# Location Dependence Derrivative
G = g.jacobian(r)
# Velocity Dependence Derrivative
H = h.jacobian(v)
```

```python
# Costate Equations of Motion
dl = -Matrix([Hamiltonian.diff(i) for i in s])
```

```python
# Fullstate Equations of Motion
dfs = Matrix([ds, dl])
```

```python
# Switching Function
S = -lv.norm()*c/m - lm + 1
```

```python
# Optimal Thrust Direction
aopt = -lv.normalized()
```

```python
# Derrivative of Fullstate
ddfs = dfs.jacobian(fs)
```

# Implementation
Initialisation parameters sourced from [*Low-Thrust Minimum Fuel Optimization in the
Circular Restricted Three-Body Problem* (Zhang et. al.)](https://home.aero.polimi.it/topputo/data/uploads/papers/articles/article-2015-2.pdf)

The nonlinear programming problem's decision vector is
$$ [t_f, \lambda_x(t_i), \lambda_y(t_i), \lambda_z(t_i), \lambda_{v_x}(t_i), \lambda_{v_y}(t_i), \lambda_{v_z}(t_i), \lambda_m(t_i)] $$

```python
import sys
sys.path.append('../')
from trajectory.space import CRTBP
from numpy import *
from PyGMO import *
```

```python
# Parameters
mu  = 1.21506683e-2 # Gravitational parameter
T   = 10. # Maximum thrust [N]
Isp = 2000. # Specific impulse
eps = 1. # Homotopy paramter (energy-minimisation)
```

```python
# Initial state
si = array([-0.019488511458668, -0.016033479812051, 0,
            8.918881923678198, -4.081793688818725, 0,
            1000], float)
# Target state
st = array([0.823385182067467, 0, -0.022277556273235,
            0, 0.134184170262437, 0,
            1000], float)
# Decision variable guess
di = array([8.6,                            # Time of flight [days]
            5.616017, 32.875896, -0.094522, # Position costates
            -0.101606, 0.044791, -0.000150, # Velocity costates
            0.133266], float)               # Mass costate
```

```python
# Intialise the problem
prob = CRTBP(mu, T, Isp, eps, si, st)
```

```python
algo = algorithm.scipy_slsqp(max_iter=20, screen_output=True)
algo = algorithm.mbh(algo, screen_output=True)
```

```python
pop = population(prob, 1)
```

    /usr/lib/python2.7/dist-packages/scipy/integrate/_ode.py:1035: UserWarning: dop853: larger nmax is needed
      self.messages.get(idid, 'Unexpected idid=%s' % idid))

```python

```


# Optimización de funciones escalares diferenciables con `SymPy`

> - Mediante optimización se obtienen soluciones elegantes tanto en teoría como en ciertas aplicaciones.
> - La teoría de optimización usa elementos comenzando con cálculo elemental y álgebra lineal básica, y luego se extiende con análisis funcional y convexo.
> - Las aplicaciones en optimización involucran ciencia, ingeniería, economía, finanzas e industria.
> - El amplio y creciente uso de la optimización lo hace escencial para estudiantes y profesionales de cualquier rama de la ciencia y la tecnología.

**Referencia:**
- http://www.math.uwaterloo.ca/~hwolkowi//henry/reports/talks.d/t06talks.d/06msribirs.d/optimportance.shtml

Algunas aplicaciones son:

1. Ingeniería
 - Encontrar la composición de equilibrio de una mezcla de diferentes átomos.
 - Planeación de ruta para un robot (o vehículo aéreo no tripulado).
 - Planeación de la mano de obra óptima en una construcción o planta de producción.
2. Distribución óptima de recursos.
 - Distribución de rutas de vuelo.
 - Encontrar una dieta óptima.
 - Planeación de ruta óptima.
3. Optimización financiera
 - Administración de riesgos.
 - Portafolios de inversión.

En esta clase veremos aspectos básicos de optimización. En específico, veremos cómo obtener máximos y mínimos de una función escalar de una variable (como en cálculo diferencial).

___
## 0. Librerías que usaremos

Como ya dijimos en la primer clase `python` es el lenguaje de programación (el cual es de alto nivel). Sin embargo, `python` solo tiene unos pocos comandos primitivos y para hacer más fácil su uso en nuestras actividades de simulación en ingeniería, otras personas ya han escrito ciertas librerías por nosotros.

### 0.1 `NumPy`

`NumPy` (Numerical Python) es la librería fundamental para computación científica (numérica) con `Python`. Contiene, entre otras cosas:
- un objeto tipo arreglo N-dimensional muy poderoso
- funciones sofisticadas
- funciones de álgebra lineal, transformada de Fourier y números aleatorios.

Por lo anterior, `NumPy` es de amplio uso entre la comunidad científica e ingenieril (por su manejo de cantidades vectoriales). De la misma manera, se usa para guardar datos. Para nuestros propósitos, se puede usar libremente.

**Referencia:**
- http://www.numpy.org/

`NumPy` ya viene incluido en la instalación estándar de Anaconda por defecto. Para comenzar a usarlo, solo debemos de importarlo:

```python
# importar la librería numpy

```

### 0.2 `SymPy`

`SymPy` (Symbolic Python) es una librería de `Python` para matemáticas simbólicas. Su objetivo es convertirse en un sistema de álgebra computacional con las mejores características, manteniendo el código lo más simple posible para que sea comprensible.

**Referencia:**
- http://www.sympy.org/en/index.html

`SymPy` ya viene incluido en la instalación estándar de Anaconda por defecto. Para comenzar a usarlo, solo debemos de importarlo:

```python
# importar la librería sympy

```

```python
# imprimir en formato latex

```

La funcionalidad de imprimir en formato LaTex que nos da `SymPy` mediante el proyecto `mathjax` hace de `SymPy` una herramienta muy atractiva...

Notar que en `SymPy` y en `NumPy` existen funciones con el mismo nombre, pero reciben tipos de datos diferentes...

```python

```

```python
# diferencias de funciones de sympy y numpy

```

```python

```

```python

```

```python

```

Explicar el uso de la sintaxis `from numpy import *` y sus peligros (no recomendable).

```python
# importar con * y ver que pasa
# from numpy import *
# from sympy import *
# No recomendado
```

### 0.3 `PyPlot` de `matplotlib`

El módulo `PyPlot` de la librería `matplotlib` contiene funciones que nos permite generar una gran cantidad de gráficas rápidamente. Las funciones de este módulo están escritas con el mismo nombre que las funciones para graficar en `Matlab`.

**Referencia:**
- https://matplotlib.org/api/pyplot_summary.html

```python
# importar matplotlib.pyplot

# comando para que las gráficas salgan en la misma ventana

```

Ya que revisamos todas las librerías que usaremos, empecemos con la clase como tal...

___
Basamos todos los resultados en los siguientes teoremas:

## 1. Teorema de Fermat (análisis)

Si una función $f(x)$ alcanza un máximo o mínimo local en $x=c$, y si la derivada $f'(c)$ existe en el punto $c$, entonces $f'(c) = 0$.

### Ejemplo

Sabemos que la función $f(x)=x^2$ tiene un mínimo global en $x=0$, pues

$$f(x)=x^2\geq0,\qquad\text{y}\qquad f(x)=x^2=0 \qquad\text{si y solo si}\qquad x=0.$$

```python
# declarar la variable real x

```

```python
# declarar ahora f=x^2 y mostrar

```

```python
# derivar f respecto a x y mostrar

```

```python
# resolver f'(x)=0 y mostrar soluciones

```

```python

```

Veamos la gráfica...

```python
# convertir f e una función que se pueda evaluar numéricamente (función lambdify de la librería sympy)

```

```python
# Coordenadas x (abscisas)

```

```python
# graficar
# Crear ventana de graficos y damos medidas de la ventana

# Sirve para hacer el grafico y determinar sus caracteristicas

# Nombre del eje x de la grafica

# Nombre del eje y

# Sirve para poner las etiquetas de las graficas

# Sirve para poner la cuadricula

```

Ver diferencias entre f y f_num

```python
# intentar evaluar f y f_num

```

```python

```

```python

```

**Otra manera de hacer lo anterior**

Concepto de función...

```python

```

```python

```

```python

```

```python

```

```python

```

```python
# graficar
# Crear ventana de graficos y damos medidas de la ventana

# Sirve para hacer el grafico y determinar sus caracteristicas
#plt.plot(xnum, funcion_de_clase(xnum), 'k', label='$y=x^2$')

# Nombre del eje x de la grafica

# Nombre del eje y

# Sirve para poner las etiquetas de las graficas

# Sirve para poner la cuadricula

```

El converso del teorema anterior no es cierto.

### Actividad
Considere $g(x)=x^3$.
- Usando `sympy`, muestre que $g'(0)=0$.
- Sin embargo, descartar que $x=0$ es un extremo de $g(x)$ viendo su **gráfica**.

```python
# Declarar la variable simbolica x

```

```python
# Definimos funcion g(x)

```

```python
# Derivamos g(x)

```

```python
# Puntos criticos

```

```python
# graficar
# Crear ventana de graficos y damos medidas de la ventana

# Sirve para hacer el grafico y determinar sus caracteristicas

# Nombre del eje x de la grafica

# Nombre del eje y

# Sirve para poner las etiquetas de las graficas

# Sirve para poner la cuadricula

```

## 2. Criterio de la segunda derivada

Sea $f(x)$ una función tal que $f’(c)=0$ y cuya segunda derivada existe en un intervalo abierto que contiene a $c$.
- Si $f’’(c)>0$, entonces $f(c)$ es un mínimo relativo.
- Si $f’’(c)<0$, entonces $f(c)$ es un máximo relativo.
- Si $f’’(c)=0$, entonces el criterio no decide.

### Ejemplo

Mostrar, usando `sympy`, que la función $f(x)=x^2$ tiene un mínimo relativo en $x=0$.

Ya vimos que $f'(0)=0$. Notemos que:

```python

```

```python
# Sacamos la segunda derivada

```

```python

```

Por tanto, por el criterio de la segunda derivada, $f(0)=0$ es un mínimo relativo (en efecto, el mínimo global).

### Ejemplo
¿Qué pasa con $g(x)=x^3$ al intentar utilizar el criterio de la segunda derivada? (usar `sympy`).

```python

```

```python

```

```python

```

Como $g''(0)=0$ entonces el criterio de la segunda derivada no concluye.

### Actividad

¿Qué pasa con $h(x)=x^4$ al intentar utilizar el criterio de la segunda derivada?.

```python

```

```python

```

```python

```

```python

```

```python

```

## 3. Método para determinar extremos absolutos de una función continua y=f(x) en [a,b]

- Determinar todos los valores críticos $c_1, c_2, c_3, \dots, c_n$ en $(a,b)$.
- Evaluar $f$ en todos los valores críticos y en los extremos $x=a$ y $x=b$.
- El más grande y el más pequeño de los valores de la lista $f(a), f(b), f(c_1), f(c_2), \dots, f(c_n)$ son el máximo absoluto y el mínimo absoluto, respectivamente, de f en el intervalo [a,b].

### Ejemplo

Determinar los extremos absolutos de $f(x)=x^2-6x$ en $\left[0,5\right]$.

Obtenemos los puntos críticos de $f$ en $\left[0,5\right]$:

```python

```

```python
# Derivamos f

```

```python

```

Evaluamos $f$ en los extremos y en los puntos críticos:

```python

```

Concluimos que el máximo absoluto de $f$ en $\left[0,5\right]$ es $0$ y se alcanza en $x=0$, y que el mínimo absoluto es $-9$ y se alcanza en $x=3$.

```python
# graficar

# Crear ventana de graficos y damos medidas de la ventana

# Sirve para hacer el grafico y determinar sus caracteristicas

# Nombre del eje x de la grafica

# Nombre del eje y

# Sirve para poner las etiquetas de las graficas

# Sirve para poner la cuadricula

```

### Actividad
Determinar los valores extremos absolutos de $h(x)=x^3-3x$ en $\left[-2.2,1.8\right]$, usando `sympy`. Mostrar en una gráfica.

```python

```

```python

```

```python

```

```python

```

```python

```

```python

```

```python

```

### En varias variables...

El procedimiento es análogo.

Si una función $f:\mathbb{R}^n\to\mathbb{R}$ alcanza un máximo o mínimo local en $\boldsymbol{x}=\boldsymbol{c}\in\mathbb{R}^n$, y $f$ es diferenciable en el punto $\boldsymbol{x}=\boldsymbol{c}$, entonces $\left.\frac{\partial f}{\partial \boldsymbol{x}}\right|_{\boldsymbol{x}=\boldsymbol{c}}=\boldsymbol{0}$ (todas las derivadas parciales en el punto $\boldsymbol{x}=\boldsymbol{c}$ son cero).

**Criterio de la segunda derivada:** para ver si es máximo o mínimo, se toma la segunda derivada (matriz jacobiana) y se verifica definición negativa o positiva, respectivamente.

Si se restringe a cierta región, hay ciertas técnicas. La más general, pero también la más compleja es la de **multiplicadores de Lagrange**.

**Ejemplo:** hacer a mano a la vez para corroborar...

```python
sym.var('x y')
x, y
```

```python
def f(x, y):
    return x**2 + y**2
```

```python
dfx = sym.diff(f(x,y), x)
dfy = sym.diff(f(x,y), y)
dfx, dfy
```

```python
xy_c = sym.solve([dfx, dfy], [x, y])
xy_c
```

```python
x_c, y_c = xy_c[x], xy_c[y]
x_c, y_c
```

```python
d2fx = sym.diff(f(x,y), x, 2)
d2fy = sym.diff(f(x,y), y, 2)
dfxy = sym.diff(f(x,y), x, y)

Jf = sym.Matrix([[d2fx, dfxy], [dfxy, d2fy]])
Jf.eigenvals()
```

```python
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
```

```python
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')

x = np.linspace(-2, 2, 100)
y = x
X, Y = np.meshgrid(x, y)

ax.plot_surface(X, Y, f(X, Y))
ax.plot([x_c], [y_c], [f(x_c,y_c)], '*r')
```

# Anuncios parroquiales

## 1. [Curso gratis sugerido](https://www.kaggle.com/learn/python)

<footer id="attribution" style="float:right; color:#808080; background:#fff;">
Created with Jupyter by Esteban Jiménez Rodríguez.
</footer>


# Tutorial 1.c.2 Conformal Calibration: From any Prediction to Calibrated Probability

In the previous tutorial we used conformal (interval) prediction to convert any prediction type into intervals with correct coverage. In this tutorial we will take this a step further and convert any prediction type into distributions that are calibrated. Recall that a distribution prediction can be represented as a random variable $F$ that takes values in the set of cumulative distribution functions $F: \mathbb{R} \to [0, 1]$, and we say that the distribution prediction is calibrated if
\begin{align}
\mathrm{Pr}[F(Y) \leq c] = \Pr[Y \leq F^{-1}(c)] = c, \forall c  \label{eq:calibration} \tag{1}
\end{align}
where $Y$ is the label. In this formula, $F$ should be interpreted as a random variable (i.e. a randomly sampled prediction) and $Y$ is the associated true label of the randomly sampled prediction. Intuitively, $F^{-1}(c)$ is the $c$-th quantile of the predicted CDF, and $Y$ is below the $c$-th quantile exactly $c$-proportion of times.

This is a generalization of what interval prediction in the previous tutorial because distributions contain strictly more information than intervals. As shown in (cite), if $F$ is a calibrated distribution prediction, then the interval $[F^{-1}(c/2), F^{-1}(1-c/2)]$ is an interval prediction with correct $c$-coverage. Formally if Eq.\eqref{eq:calibration} is true then
\begin{align}
\Pr \left[ Y \in [F^{-1}(c/2), F^{-1}(1-c/2)] \right] = c
\end{align}

Conformal calibration (cite) is a method of converting any prediction type into calibrated probabilities. The intuition behind the algorithm is very similar to conformal (interval) prediction in the previous tutorial. In this tutorial we will not go into details of how conformal calibration works, but how to implement it in torchuq. The main workhorse for conformal calibration is the class ``torchuq.transform.conformal.ConformalCalibrator``. We first import it the class, as well as some test prediction data (same as tutorial 1.a).

```python
from matplotlib import pyplot as plt
import torch

import sys
sys.path.append('../..')   # Include the directory that contains the torchuq package
from torchuq.transform.conformal import ConformalCalibrator

reader = torch.load('pretrained/boston_pretrained.tar')  # Load the pretrained predictions
val_pred = reader['predictions_quantile'][:50]
val_labels = reader['labels'][:50]
test_pred = reader['predictions_quantile'][50:]
test_labels = reader['labels'][50:]
```

``ConformalCalibrator`` has exactly the same interface as ``ConformalIntervalPredictor`` in the previous tutorial. Specifically, there are three functions that you need to know

1. Constructor: ``calibrator = ConformalCalibrator(input_type='quantile', interpolation='linear')``. there is only one required argument, which is the prediction type, it is one of the types introduced in tutorial 1.a.

2. Train: ``ConformalCalibrator.train(val_preds, val_labels)`` trains the conformal predictor based on validation predictions and validation labels

3. Test: ``test_distribution = ConformalCalibrator.__call__(test_preds)`` outputs the calibrated distribution predictions.

```python
# ConformalCalibrator takes an argument called interpolation. For details of what this option means please see (cite).
# In general, use 'random' if you want better calibration, use 'naf' if you want better log likelihood (but worse numerical stability and computation speed)
# Use 'linear' for a good trade-off (default option)
calibrator = ConformalCalibrator(input_type='quantile', interpolation='random')
calibrator.train(val_pred, val_labels)
test_distribution = calibrator(test_pred)
```

```python
from torchuq.evaluate import distribution, interval
distribution.plot_reliability_diagram(test_distribution, test_labels);
```

We can also convert the distribution to intervals with the function ``torchuq.transform.direct.distribution_to_interval``, which makes the simplest transformation introduced in the beginning of this tutorial (i.e. mapping CDF $F$ to interval $F^{-1}(c/2), F^{-1}(1-c/2)$. We visualize the intervals and demonstrate that these intervals appear to have the correct coverage.

```python
from torchuq.transform.direct import distribution_to_interval

converted_interval = distribution_to_interval(test_distribution, confidence=0.95)
interval.plot_interval_sequence(converted_interval, test_labels)
```

## Online Calibration

Similar to conformal interval prediction, we can also use conformal calibration in the online setup. The usage is almost identical to the previous tutorial. Here is an example.

At each step, we only make a single distribution prediction (i.e. a batch of distribution prediction with ``batch_size=1``). However for visualization purposes we will want to concatenate them into a single batch of distribution predictions (because all torchuq visualziation functions take as input a single batch of distribution prediction). Pytorch does not provide this function, so for convenience we implement concatenation in torchuq with the class ``ConcatDistribution``.

```python
from torchuq.transform.basic import ConcatDistribution

calibrator = ConformalCalibrator(input_type='quantile')

val_preds = reader['predictions_quantile'][:50]
val_labels = reader['labels'][:50]   # Load the true label (i.e. the ground truth housing prices)

def simulate_online(calibrator):
    # There needs to be at least 1 data point before making any prediction
    calibrator.train(val_preds[0:1], val_labels[0:1])

    prediction_history = []
    for t in range(1, 50):
        test_distribution = calibrator(val_preds[t:t+1])  # Make a prediction for the new time step

        # If this weren't a simulation, you would do something with the prediction before the label is observed

        calibrator.update(val_preds[t:t+1], val_labels[t:t+1])   # Update the calibrator based on the observed label

        prediction_history.append(test_distribution)

    # Concat the interval predictions for plotting
    prediction_history = ConcatDistribution(prediction_history)
    return prediction_history

prediction_history = simulate_online(calibrator)
distribution.plot_density_sequence(prediction_history, val_labels[1:50], smooth_bw=10)   # Smooth the resulting CDF a bit for better visual appeal
distribution.plot_reliability_diagram(prediction_history, val_labels[1:50]);
```

Observe that we get much better calibration in the online setup. The reason for this is very interesting and sutble. We will not go into detail of why this is and refer interested readers to (cite).

```python

```