```python
import sympy
from sympy.abc import X, B, C, D     # Establish our variable names
import numpy as np
import matplotlib.pyplot as plt
import scipy
```

```python
%matplotlib inline
```

```python
def interpolate(conds):
    """
    Finds the polynomial of minimal degree that solves the given Hermite conditions.

    conds is a list of the form
      [(x1, r1, v1), (x2, r2, v2), ...]
    where the polynomial p is to satisfy p^(r_1) (x_1) = v_1, and so on.
    """
    # the degree will be one less than the number of conditions
    n = len(conds)

    # generate a symbol for each coefficient
    A = [sympy.Symbol("a[%d]" % i) for i in range(n)]

    # generate the desired polynomial symbolically
    P = sum([A[i] * X**i for i in range(n)])

    # generate the equations the polynomial must satisfy
    #
    # for each (x, r, v), sympy evaluates the rth derivative of P wrt X,
    # substitutes x in for X, and requires that this equals v.
    EQNS = [sympy.diff(P, X, r).subs(X, x) - v for x, r, v in conds]

    # solve this system.
    SOLN = sympy.solve(EQNS, A)

    return P.subs(SOLN)

```

```python
interpolate([(1, 0, 1), (1, 1, 2)])
```

    2*X - 1

```python
# We fix the values of a0, b0, c0.
a0 = 0
b0 = 3
c0 = 1

# We require p(a0) = 0, p(b0) = 0, p(c0) = 1, p'(c0) = 0.
BASIC_CONDS = [(a0, 0, 0), (b0, 0, 0), (c0, 0, 1), (c0, 1, 0)]

def c_degen(n, residue):
    """
    Give Hermite conditions for order of vanishing at c0 equal to `n`, with
    first nonzero residue `residue`.

    NOTE: the order `n` is in terms of f', not of f. That is, this is the amount
    of additional degeneracy to add.  This may be a source of off-by-one errors.
    """
    return [(c0, 1 + i, 0) for i in range(1, n + 1)] + [(c0, n + 2, residue)]

def b_degen(n, residue):
    """
    Give Hermite conditions for order of vanishing at b0 equal to `n`, with
    first nonzero residue `residue`.
    """
    return [(b0, i, 0) for i in range(1, n + 1)] + [(b0, n + 1, residue)]

def poly_with_degens(nc=0, nb=0, residue_c=3, residue_b=3):
    """
    Give unique polynomial with given degeneracies for this MVT problem.

    `nc` is the order of vanishing of f' at c0, with first nonzero residue `residue_c`.
    `nb` is the order of vanishing of f at b0, with first nonzero residue `residue_b`.
    """
    conds = BASIC_CONDS + c_degen(nc, residue_c) + b_degen(nb, residue_b)
    return interpolate(conds)
```

```python
poly_with_degens()
```

    11*X**5/16 - 21*X**4/4 + 113*X**3/8 - 65*X**2/4 + 123*X/16

```python
def smoother_interpolate(conds, ctrl_point=2, order=1, interval=(0,4)):
    """
    Find the polynomial of minimal degree that interpolates the Hermite
    conditions in `conds`, and whose behavior at `ctrl_point` minimizes the L2
    norm on `interval` of its derivative.
    """
    # Add the symbolic point to the conditions.
    # Recall that D is a sympy variable
    new_conds = conds + [(ctrl_point, order, D)]

    # Find the polynomial interpolating `new_conds`, symbolic in X *and* D
    P = interpolate(new_conds)

    # Compute L2 norm of the derivative on `interval`
    L2 = sympy.integrate(sympy.diff(P, X)**2, (X, *interval))

    # Take the first critical point of the L2 norm with respect to D
    SOLN = sympy.solve(sympy.diff(L2, D), D)[0]

    # Substitute the minimizing solution in for D and return
    return P.subs(D, SOLN)

def smoother_poly_with_degens(nc=0, nb=0, residue_c=3, residue_b=3):
    """
    Give unique polynomial with given degeneracies for this MVT problem whose
    derivative on (0, 4) has minimal L2 norm.

    `nc` is the order of vanishing of f' at c0, with first nonzero residue `residue_c`.
    `nb` is the order of vanishing of f at b0, with first nonzero residue `residue_b`.

    """
    conds = BASIC_CONDS + c_degen(nc, residue_c) + b_degen(nb, residue_b)
    return smoother_interpolate(conds)

```

```python
sympy.N(smoother_poly_with_degens())
```

    -0.290763858423069*X**6 + 3.30437472580762*X**5 - 13.9729157526921*X**4 + 27.5001374874612*X**3 - 25.8452073279613*X**2 + 10.3043747258076*X

```python
import matplotlib.pyplot as plt
import numpy as np

def basic_plot(F, n=300, outfile="basic_plot.pdf"):
    fig = plt.figure(figsize=(7, 7))
    ax = fig.add_subplot(1, 1, 1)
    b1d = np.linspace(-0.5, 4.5, n)
    f = sympy.lambdify(X, F)(b1d)
    ax.plot(b1d,f,'k')
    ax.set_aspect('equal')
    ax.grid(True)
    ax.set_xlim([-.5, 4.5])
    ax.set_ylim([-1, 5])
    ax.plot([0, c0, b0],[0, F.subs(X,c0),F.subs(X,b0)],'ko')
    fig.savefig(outfile, bbox_inches='tight')
```

```python
Fbasic = poly_with_degens()
basic_plot(Fbasic, outfile="basic_plot_one.png")
```

```python
interval = (0, 4)
sympy.N(sympy.integrate(sympy.diff(poly_with_degens(), X)**2, (X, *interval)))
```

    1865.15411706349

```python
Fsmooth_basic = smoother_poly_with_degens()
basic_plot(Fsmooth_basic, outfile="basic_plot_two.png")
```

```python
interval = (0, 4)
sympy.N(sympy.integrate(sympy.diff(smoother_poly_with_degens(), X)**2, (X, *interval)))
```

    41.1612799050325

```python
def abscissa_plot(F, n=300, outfile="abscissa_plot.pdf"):
    # Compute the derivative of f
    DF = sympy.diff(F,X)

    # Define CAP_F --- "capital F"
    #
    # this is (f(b) - f(0))/(b - 0) - f'(c).
    CAP_F = (F.subs(X, B) - F.subs(X, 0)) / (B - 0) - DF.subs(X, C)

    # build the mesh
    b1d = np.linspace(-.5, 4.5, n)
    b2d, c2d = np.meshgrid(b1d, b1d)

    # compute CAP_F within the mesh
    cap_f_mesh = sympy.lambdify((B, C), CAP_F)(b2d, c2d)

    # restrict attention to below the diagonal --- we require c < b
    # (although the mas inequality looks reversed in this perspective)
    valid_cap_f_mesh = scipy.ma.array(cap_f_mesh, mask=c2d>b2d)

    # Set up plot basics
    fig = plt.figure(figsize=(7, 7))
    ax = fig.add_subplot(1, 1, 1)
    ax.set_aspect('equal')
    ax.grid(True)
    ax.set_xlim([-.5, 4.5])
    ax.set_ylim([-.5, 4.5])

    # plot the contour
    ax.contour(b2d, c2d, valid_cap_f_mesh, [0], colors='k')

    # plot a diagonal line representing the boundary
    ax.plot(b1d,b1d,'k--')

    # plot the guaranteed point
    ax.plot(b0,c0,'ko')

    fig.savefig(outfile, bbox_inches="tight")
```

```python
abscissa_plot(Fbasic, outfile="abs_plot_one.png")
```

```python
abscissa_plot(Fsmooth_basic, outfile="abs_plot_two.png")
```

```python
F3 = smoother_poly_with_degens(nc=1, residue_b=-3)
basic_plot(F3, outfile="basic_plot_F3.png")
```

```python
abscissa_plot(F3, outfile="abs_plot_f3.png")
```

```python

```


# SymPy: Open Source Symbolic Mathematics

This notebook uses the [SymPy](http://sympy.org) package to perform symbolic manipulations,
and combined with numpy and matplotlib, also displays numerical visualizations of symbolically
constructed expressions.

We first load sympy printing and plotting support, as well as all of sympy:

```
%load_ext sympyprinting
%pylab inline

from __future__ import division
import sympy as sym
from sympy import *
x, y, z = symbols("x y z")
k, m, n = symbols("k m n", integer=True)
f, g, h = map(Function, 'fgh')
```

    Welcome to pylab, a matplotlib-based Python environment [backend: module://IPython.zmq.pylab.backend_inline].
    For more information, type 'help(pylab)'.

Elementary operations

```
Rational(3,2)*pi + exp(I*x) / (x**2 + y)
```

```
exp(I*x).subs(x,pi).evalf()
```

```
e = x + 2*y
```

```
srepr(e)
```

    Add(Symbol('x'), Mul(Integer(2), Symbol('y')))

```
exp(pi * sqrt(163)).evalf(50)
```

Algebra

```
eq = ((x+y)**2 * (x+1))
eq
```

```
expand(eq)
```

```
a = 1/x + (x*sin(x) - 1)/x
a
```

```
simplify(a)
```

```
eq = Eq(x**3 + 2*x**2 + 4*x + 8, 0)
eq
```

```
solve(eq, x)
```

    [-2, -2⋅ⅈ, 2⋅ⅈ]

```
a, b = symbols('a b')
Sum(6*n**2 + 2**n, (n, a, b))
```

Calculus

```
limit((sin(x)-x)/x**3, x, 0)
```

```
(1/cos(x)).series(x, 0, 6)
```

```
diff(cos(x**2)**2 / (1+x), x)
```

```
integrate(x**2 * cos(x), (x, 0, pi/2))
```

```
eqn = Eq(Derivative(f(x),x,x) + 9*f(x), 1)
display(eqn)
dsolve(eqn, f(x))
```

# Illustrating Taylor series

We will define a function to compute the Taylor series expansions of a symbolically defined expression at
various orders and visualize all the approximations together with the original function

```
# You can change the default figure size to be a bit larger if you want,
# uncomment the next line for that:
#plt.rc('figure', figsize=(10, 6))
```

```
def plot_taylor_approximations(func, x0=None, orders=(2, 4), xrange=(0,1), yrange=None, npts=200):
    """Plot the Taylor series approximations to a function at various orders.

    Parameters
    ----------
    func : a sympy function
    x0 : float
      Origin of the Taylor series expansion.  If not given, x0=xrange[0].
    orders : list
      List of integers with the orders of Taylor series to show.  Default is (2, 4).
    xrange : 2-tuple or array.
      Either an (xmin, xmax) tuple indicating the x range for the plot (default is (0, 1)),
      or the actual array of values to use.
    yrange : 2-tuple
      (ymin, ymax) tuple indicating the y range for the plot.  If not given,
      the full range of values will be automatically used.
    npts : int
      Number of points to sample the x range with.  Default is 200.
    """
    if not callable(func):
        raise ValueError('func must be callable')
    if isinstance(xrange, (list, tuple)):
        x = np.linspace(float(xrange[0]), float(xrange[1]), npts)
    else:
        x = xrange
    if x0 is None: x0 = x[0]
    xs = sym.Symbol('x')
    # Make a numpy-callable form of the original function for plotting
    fx = func(xs)
    f = sym.lambdify(xs, fx, modules=['numpy'])
    # We could use latex(fx) instead of str(), but matploblib gets confused
    # with some of the (valid) latex constructs sympy emits.  So we play it safe.
    plot(x, f(x), label=str(fx), lw=2)
    # Build the Taylor approximations, plotting as we go
    apps = {}
    for order in orders:
        app = fx.series(xs, x0, n=order).removeO()
        apps[order] = app
        # Must be careful here: if the approximation is a constant, we can't
        # blindly use lambdify as it won't do the right thing.  In that case,
        # evaluate the number as a float and fill the y array with that value.
        if isinstance(app, sym.numbers.Number):
            y = np.zeros_like(x)
            y.fill(app.evalf())
        else:
            fa = sym.lambdify(xs, app, modules=['numpy'])
            y = fa(x)
        tex = sym.latex(app).replace('$', '')
        plot(x, y, label=r'$n=%s:\, %s$' % (order, tex) )

    # Plot refinements
    if yrange is not None:
        plt.ylim(*yrange)
    grid()
    legend(loc='best').get_frame().set_alpha(0.8)
```

With this function defined, we can now use it for any sympy function or expression

```
plot_taylor_approximations(sin, 0, [2, 4, 6], (0, 2*pi), (-2,2))
```

```
plot_taylor_approximations(cos, 0, [2, 4, 6], (0, 2*pi), (-2,2))
```

This shows easily how a Taylor series is useless beyond its convergence radius, illustrated by
a simple function that has singularities on the real axis:

```
# For an expression made from elementary functions, we must first make it into
# a callable function, the simplest way is to use the Python lambda construct.
plot_taylor_approximations(lambda x: 1/cos(x), 0, [2,4,6], (0, 2*pi), (-5,5))
```


```python
%matplotlib inline
%config InlineBackend.figure_format = 'retina'

%load_ext autoreload
%autoreload 2

import numpy as np
import matplotlib.pyplot as plt
import matplotlib.pylab as pylab
from simplified_monorotor import Monorotor
import plotting
import testing
import trajectories

pylab.rcParams['figure.figsize'] = 10,10
```

# PD controller

#### TODO 1 - Implement PD controller

Implement the PD Controller math in `thrust_control`:

$$
\begin{align}
e &= z_{\text{target}} - z_{\text{actual}} \\
\dot{e} &= \dot{z}_{\text{target}} - \dot{z}_{\text{actual}} \\
\bar{u}_1 &= k_p e + k_d \dot{e} \\
u_1 &= m(g - \bar{u}_1)
\end{align}
$$

```python
class PDController:

    def __init__(self, k_p, k_d, m):
        self.k_p = k_p
        self.k_d = k_d
        self.vehicle_mass = m
        self.g = 9.81

    def thrust_control(self,
                z_target,
                z_actual,
                z_dot_target,
                z_dot_actual,
                z_dot_dot_ff=0.0): # IGNORE this for now.
        #
        # TODO
        #   implement PD controller math shown above
        #   and return a commanded thrust.
        #
        #   You can ignore the z_dot_dot_ff parameter for now.
        e = z_target - z_actual
        e_dot = z_dot_target - z_dot_actual
        u_bar = self.k_p * e + self.k_d * e_dot
        u = self.vehicle_mass * (self.g - u_bar)

        return u

testing.pd_controller_test(PDController, feed_forward=False)
```

    Tests pass

#### TODO 2 - Adjust parameters

Start by running the code below with K_D = 0.0. This will remind you what a P-controlled trajectory looks like. Then try slowly increasing K_D to 0.5, 1.0, 2.0, etc...

What value of K_D gives a reasonable trajectory?

Is there a problem with setting K_D too high?

```python
MASS_ERROR = 1.5
K_P = 20.0
K_D = 5

# preparation
drone = Monorotor()
perceived_mass = drone.m * MASS_ERROR
controller = PDController(K_P, K_D, perceived_mass)

# generate trajectory
total_time = 3.0
dt = 0.001
t=np.linspace(0.0,total_time,int(total_time/dt))
z_path= -np.ones(t.shape[0])
z_dot_path = np.zeros(t.shape[0])

# run simulation
history = []
for z_target, z_dot_target in zip(z_path, z_dot_path):
    z_actual = drone.z
    z_dot_actual = drone.z_dot
    u = controller.thrust_control(z_target, z_actual,
                                  z_dot_target, z_dot_actual)
    drone.thrust = u
    drone.advance_state(dt)
    history.append(drone.X)

# generate plots
z_actual = [h[0] for h in history]
plotting.compare_planned_to_actual(z_actual, z_path, t)
```

[Solution](/notebooks/PD%20Controller%20Solution.ipynb)

```python

```


# Generating continuous random variables

## The integral transformation probability theorem ##

In the second lecture we introduced several random number generators for the discrete uniform distribution defined on a given set $\{0.. m-1\}$, for a suitable $m \in \mathbb N$, and we also saw that these generators can be easily adapted to the uniform distribution on the continuous interval $[0, 1)$. In the third lecture we introduced the integral transformation technique in order to use any such generator in order to simulate any discrete distribution. In this lecture we will see how this technique can be easily adapted to the case of continuous distributions.

First of all, consider the following theoretical result.

**Theorem**: Let $X$ be a continuous random variable whose cumulative distribution function is $F_X$. The random variable $Y = F_X(X)$ follows a uniform distribution over $[0, 1]$.

_Proof_: Denote by $F_Y$ the the cumulative distribution of $Y$, and remark that $Y$ assumes specifications in $[0, 1]$. Note also that the continuity and monotonicity properties of a continuous c.d.f. ensure that $F_X$ is invertible. Now, for each $x$ in the latter set,

$$
F_Y(x) = \mathrm P(Y \leq x)
       = \mathrm P(F_X(X) \leq x)
       = \mathrm P(F_X^{-1}(F_X(X)) \leq F_X^{-1}(x))
       = \mathrm P(X \leq F_X^{-1}(x))
       = F_X(F_X^{-1}(x)) = x.
$$

Thus $F_X$ coincide with the c.d.f. of the random distribution over $[0, 1]$, and the theorem is proven. ∎

Therefore, by abuse of notation we can write $F_X(X) = U$, where $U$ is uniformly distributed over $[0, 1]$. Pushing further this abuse of notation, this theorem suggests that $X = F_X^{-1}(U)$, that is $F_X^{-1}(U)$ is distributed as $X$. This is precisely shown as follows.

**Theorem**: Let $X$ and $F_X$ denote a continuous random variable and its cumulative distribution function, respectively. Furthermore, let $U$ denote a random variable uniformly distributed on $[0,1]$ and define $Y=F_X^{-1}(U)$. Then $Y$ is a random variable having the same distribution of $X$.

_Proof_: In order to prove the thesis it is sufficient to show that the cumulative distribution functions of $X$ and $Y$ coincide. Now, given a generic $y$ in the support of $Y$,

$$
F_Y(y) = \mathrm P(Y \leq y)
       = \mathrm P(F_X^{-1}(U) \leq y)
       = \mathrm P(F_X(F_X^{-1}(U)) \leq F_X(y))
       = \mathrm P(U\leq F_X(y))
       = F_X(y)
$$

where the third equality derive from the fact that cumulative distribution functions are monotone nondecreasing, while tha last two equalities exploit the definition of inverse function and the fact that $\mathrm P(U \leq u) = u$. ∎

This result suggests a general method for building a pseudorandom sequence $\{x_0, x_1, \dots, x_i,  \dots\}$ with reference to a generic continuous random variable $X$ staring from an analogous sequence $\{u_0, u_1, \dots, u_i,  \dots\}$ for the uniform distribution on $[0,1)$: it will be sufficient to define $x_i = F_X^{-1}(u_i)$. Said in other words, the generation of each pseudorandom value $x$ for $X$ will be done in two stages: firstly, a pseudorandom value $u$ for the continuous distribution in $[0,1)$ will be drawn, and subsequently the procedure will return $x = F_X^{-1}(u)$.

The following panel graphically shows this process, using as reference an exponential distribution whose paramenter $\lambda$ can be tuned using a slider: clicking the button will generate a new value  for $u \in [0, 1]$ and obtain the corresponding $x \in \mathbb R^+$. If you peek into the code (and you should!) don't worry about how the actual value for $x$ is computed: we'll see it in a couple of paragraphs.

```python
%matplotlib notebook

import matplotlib.pyplot as plt
import matplotlib.patches as patches
import numpy as np
import math
from scipy import random
from ipywidgets import widgets, interact
from IPython.display import display, clear_output

def F_exponential(x, _lambda):
    return 1 - math.e** (-_lambda * x)

fig_exp_ex, ax_exp_ex = plt.subplots()

button_change_u = widgets.Button(description="Change u")
lambda_chooser = widgets.FloatSlider(min=.1, max=5, step=.1, value=2)

u_label = widgets.Label()
x_label = widgets.Label()

def graph_invert_F_exp(_lambda):
    x = np.arange(0, 5, 0.1)
    y = F_exponential(x, _lambda)

    x_val = -float(1) / _lambda * math.log(1 - u_val)
    x_label.value = 'x={:.2f}'.format(x_val)
    f_val = F_exponential(x_val, _lambda)

    ax_exp_ex.clear()
    ax_exp_ex.add_patch(patches.FancyArrow(0, u_val, x_val, 0,
                                       length_includes_head=True, head_length=.1))

    ax_exp_ex.add_patch(patches.FancyArrow(x_val, u_val, 0, -u_val, head_width=0.07,
                                       length_includes_head=True, head_length=.025))
    ax_exp_ex.plot(x, y)
    plt.xlim(0, 5)
    display(fig_exp_ex)

def change_u(button):
    global u_val
    u_val = random.random()
    u_label.value = 'u={:.2f}'.format(u_val)
    clear_output()
    graph_invert_F_exp(lambda_chooser.value)

button_change_u.on_click(change_u)
change_u(button_change_u)

interact(graph_invert_F_exp, _lambda=lambda_chooser)

display(widgets.HBox((button_change_u, u_label, x_label)))
```

    <IPython.core.display.Javascript object>

## Simulating a uniform continuous distribution ##

The uniform continous distribution is defined over an interval $(a, b)$, and its c.d.f. has the following analytical form:

$$
F_X(x) = \frac{x - a}{b - a} \mathrm I_{(a, b)}(x)
$$

A direct application of the inverse transformation method leads to the following simulation algorithm:

- simulate $u$ uniformly in $[0, 1]$
- solve $u = F_X(x)$ w.r.t. $x$
- return $x$.

Note that the second step amounts to solving $u = \frac{x - a}{b - a}$, that is setting $x = a + u (b - a)$. Thus the simulation of a generic uniform continuous distribution can be done in one line of code.

The following panel uses this approach in order to generate one thousand specifications from a uniform distribution over an interval whose extremes can be picked through a slider, superimposing their empirical c.d.f. with the c.d.f. of the corresponding distribution. Data are generated anew each time that the _Simulate_ button is clicked.

```python
def ecdf(data):
    sorted = np.sort(data)
    yvals = np.arange(len(sorted))/float(len(sorted))
    return((sorted, yvals))

unif_button = widgets.Button(description='Simulate')
unif_ab_selector = widgets.FloatRangeSlider(min=0, max=10, value=(3, 5), description='$a, b$')

fig_unif_simulation, ax_unif_simulation = plt.subplots()

def uniform_continuous(a, b):
    return a + random.random() * (b - a)

def uniform_simulation(ab):
    data = [uniform_continuous(*ab) for i in range(1000)]
    vals = np.arange(0, 10, .1)
    cdfs = map(lambda x: 0 if x < ab[0] else 1 if x > ab[1] else (x-ab[0])/(ab[1]-ab[0]), vals)

    clear_output()
    ax_unif_simulation.clear()

    ax_unif_simulation.plot(vals, cdfs, 'b')

    x_ecdf, y_ecdf = ecdf(data)
    ax_unif_simulation.plot(x_ecdf, y_ecdf, 'ko', markersize=2, alpha=.7)
    #plt.ylim(0, 1.1)
    #plt.xlim(0, max(vals)*1.1)

    display(fig_unif_simulation)

unif_button.on_click(lambda b: uniform_simulation(unif_ab_selector.value))
display(unif_button)

interact(uniform_simulation, ab=unif_ab_selector)
```

    <IPython.core.display.Javascript object>

## Simulating an exponential distribution ##

Another distribution easily dealt through direct application of the integral transformation technique is the exponential one. Indeed, suppose that $X$ follows such a distribution for a value $\lambda > 0$ of the related parameter, that is $F_X(x) = 1 - \mathrm e^{-\lambda x} \mathrm I_{\mathbb R^+}(x)$. Since for each $u \in [0, 1]$ the solution w.r.t. $x$ of the equation $u = F_X(x)$ is

$$
x = -\frac{1}{\lambda} \ln (1-u),
$$

reminding that when $U$ is uniformly distributed over $[0, 1]$ the latter distribution is also that of $1-U$, a one-line simulation for the exponential family of distributions will map specifications $u$ of the random variable $U$ into $x = -\frac{1}{\lambda}\ln u$. The following panel implements the corresponding algorithm and tests it as usual through comparison of the theoretical and empirical c.d.f.s, allowing to dynamically select a value for the parameter $\lambda$ and to restart the simulation.

```python
exp_button = widgets.Button(description='Simulate')
exp_l_selector = widgets.FloatSlider(min=0.1, max=5, value=3, description='$\lambda$')

fig_exp_simulation, ax_exp_simulation = plt.subplots()

def exponential(_lambda):
    return -1. * math.log(random.random()) / _lambda

def exponential_simulation(_lambda):
    data = [exponential(_lambda) for i in range(1000)]
    vals = np.arange(0, 10, .1)
    cdfs = map(lambda x: 0 if x < 0 else 1 - math.exp(-_lambda * x), vals)

    clear_output()
    ax_exp_simulation.clear()

    ax_exp_simulation.plot(vals, cdfs, 'b')

    x_ecdf, y_ecdf = ecdf(data)
    ax_exp_simulation.plot(x_ecdf, y_ecdf, 'ko', markersize=2, alpha=.7)

    display(fig_exp_simulation)

exp_button.on_click(lambda b: exponential_simulation(exp_l_selector.value))
display(exp_button)

interact(exponential_simulation, _lambda=exp_l_selector)
```

    <IPython.core.display.Javascript object>

## Simulating a Laplace distribution ##

A random variable $X$ follows a Laplace (or double exponential) distribution if its probability density has the following analytical form, fixed $\alpha \in \mathbb R$ and $\beta \in \mathbb R^+$:

$$
f_X(x) = \frac{1}{2\beta}e^{-\frac{|x-\alpha|}{\beta}}.
$$

The simulation of this distribution can be dealt again through the integral transformation method, although the result is not straightforward as with the previously considered distributions. First of all it is necessary to obtain the analytical form of the cumulative distribution function. If $x \geq \alpha$ we have

$$
F_X(x) = \int_{-\infty}^{x} f_X(z) \mathrm d z
       = \int_{-\infty}^{\alpha} \frac{1}{2\beta}e^{\frac{z-\alpha}{\beta}} \mathrm d z
         + \int_{\alpha}^{z} \frac{1}{2\beta}e^{\frac{-z-\alpha}{\beta}} \mathrm d z,
$$

so that substituting $y = \frac{z-\alpha}{\beta}$ we obtain

$$
F_X(x) = \int_{-\infty}^{0} \frac{1}{2}\mathrm e^{y} \mathrm d y
         + \int_{0}^{\frac{x-\alpha}{\beta}} \frac{1}{2}\mathrm e^{-y} \mathrm d y =
         1 - \frac{1}{2}\mathrm e^{-\frac{x-\alpha}{\beta}}.
$$

A simpler computation holds for $x < \alpha$:

$$
F_X(x) = \int_{-\infty}^{x} f_X(z) \mathrm d z
       = \int_{-\infty}^{x} \frac{1}{2\beta}e^{\frac{x-\alpha}{\beta}} \mathrm d x
       = \int_{-\infty}^{\frac{x-\alpha}{\beta}}\frac{1}{2}\mathrm e^{y} \mathrm d y
       = \frac{1}{2} \mathrm e^{\frac{x - \alpha}{\beta}}.
$$

The analytical form of $F_X$ differs depending on the values it assumes: it is easy to see that the last two equations respectively hold for $F_X(x) \geq 1/2$ and $F_X(x) < 1/2$, respectively. Therefore, the simulation of $X$ will require to separately handle these two cases, as in the following algorithm:

- $u$ = random()
- if $u \geq 1/2$ return the solution w.r.t. $x$ of $1 - \frac{1}{2}\mathrm e^{-\frac{x-\alpha}{\beta}}$
- else return the solution w.r.t. $x$ of $\frac{1}{2} \mathrm e^{\frac{x - \alpha}{\beta}}$

It is easy to see that the solutions of the two branches in this algorithm are, respectively, $x = \alpha - \beta \ln (2(1-u))$ and $x = \alpha + \beta \ln 2u$. The following panel implements and tests this simulation procedure.

```python
dbexp_button = widgets.Button(description='Simulate')
dbexp_alpha_selector = widgets.FloatSlider(min=-5, max=5, value=2, description='$\\alpha$')
dbexp_beta_selector = widgets.FloatSlider(min=.1, max=3, value=1, description='$\\beta$')

fig_dbexp_simulation, ax_dbexp_simulation = plt.subplots()

def dbexponential_cdf(x, alpha, beta):
    if x >= alpha:
        return 0.5 + 0.5 * (1 - math.exp(-(x-alpha)/beta))
    else:
        return 0.5 * math.exp((x-alpha)/beta)

def double_exponential(alpha, beta):
    u = random.random()
    if u >= 0.5:
        return alpha - beta * math.log(2*(1-u))
    else:
        return alpha + beta * math.log(2*u)

def dbexponential_simulation(alpha, beta):
    data = [double_exponential(alpha, beta) for i in range(1000)]
    vals = np.arange(-5, 5, .1)
    cdfs = map(lambda x: dbexponential_cdf(x, alpha, beta), vals)

    clear_output()
    ax_dbexp_simulation.clear()

    ax_dbexp_simulation.plot(vals, cdfs, 'b')

    x_ecdf, y_ecdf = ecdf(data)
    ax_dbexp_simulation.plot(x_ecdf, y_ecdf, 'ko', markersize=2, alpha=.7)
    plt.xlim(-5, 5)

    display(fig_dbexp_simulation)

dbexp_button.on_click(lambda b: dbexponential_simulation(dbexp_alpha_selector.value, dbexp_beta_selector.value))
display(dbexp_button)

_ = interact(dbexponential_simulation, alpha=dbexp_alpha_selector, beta=dbexp_beta_selector)
```

    <IPython.core.display.Javascript object>

## Simulating a normal distribution ##

The normal distribution is more difficult do deal with than the previously considered cases, because its c.d.f. doesn't have an analytical form to be exploited in order to directly rely on the inverse transformation method.

### Numerical application of the inverse transformation method ###

The negative result just highlighted does not prevent the application of the method _per se_, because the c.d.f. inversion can be carried out numerically, as done in the following panel. Focusing for now on the standard normal distribution, the simulation exploits the `optimize.broyden1` method of the `scipy` package in order to find the root of $\Phi(x)-u$, that is to numerically invert $\Phi$. Note that this simulation is time consuming, and it can take some seconds in order to generate the thousand values that give rise to the empirical c.d.f.

```python
from scipy import stats as st
from scipy import optimize as opt

def std_random_numerical():
    u = random.random()
    return opt.broyden1(lambda x: (st.norm.cdf(x) - u), [0.5,])[0]

fig_num_std_norm, ax_num_std_norm = plt.subplots()

def new_num_std_norm_sample(button):
    data = [std_random_numerical() for i in range(1000)]

    ax_num_std_norm.clear()

    x_ecdf, y_ecdf = ecdf(data)
    ax_num_std_norm.plot(x_ecdf, y_ecdf, 'ko', markersize=2, alpha=.7)
    vals = np.arange(-3, 3, .1)
    ax_num_std_norm.plot(vals, map(st.norm.cdf, vals))

button_num_std_normal = widgets.Button(description="Simulate")
button_num_std_normal.on_click(new_num_std_norm_sample)
new_num_std_norm_sample(button_num_std_normal)

display(button_num_std_normal)
```

    <IPython.core.display.Javascript object>

### The Box-Muller method ###

There is, however, the possibility to analytically simulate a standard normal distribution, using the so-called Box-Muller method. This method considers the joint density of two independent standard normal random variables $X$ and $Y$:

$$
f_{X, Y}(x,y) = f_X(x)f_Y(y)
              = \frac{1}{\sqrt{2\pi}}\mathrm e^{\frac{-x^{2}}{2}} \frac{1}{\sqrt{2\pi}}\mathrm e^{\frac{-y^{2}}{2}}
              = \frac{1}{2\pi}\mathrm e^{-\frac{x^2+y^2}{2}},
$$

and rewrites it applying the following polar transformation to the reference system:

$$\begin{align}
x &= \sqrt{d} \cos \theta, \\
y &= \sqrt{d} \sin \theta.
\end{align}$$

The Jacobian of this transformation is

$$
J =
\begin{bmatrix}
\frac{\mathrm d x}{\mathrm d d} & \frac{\mathrm d x}{\mathrm d \theta} \\
\frac{\mathrm d y}{\mathrm d d} & \frac{\mathrm d y}{\mathrm d \theta}
\end{bmatrix}
= \begin{bmatrix}
\frac{1}{2 \sqrt{d}} \cos \theta & -\sqrt{d} \sin \theta \\
\frac{1}{2 \sqrt{d}} \sin \theta & \sqrt{d} \cos \theta
\end{bmatrix},
$$

So that its determinant equals $\frac{1}{2}$, and the joint c.d.f. of $X$ and $Y$ can be written as

$$
\begin{align}
F_{X,Y}(x,y) &= \int_{-\infty}^{x}\int_{-\infty}^{y}\frac{1}{2\pi}
                    \mathrm e^{-\frac{\overline x^2+\overline y^2}{2}}\mathrm d x \mathrm d y \\
             &= \int_{0}^{d}\int_{0}^{\theta}\frac{1}{2}\frac{1}{2\pi}
                    \mathrm e^{\frac{-\overline d}{2}}\mathrm d d \mathrm d \theta \\
             &= \int_{0}^{d} \frac{1}{2}\mathrm e^{-\frac{\overline d}{2}} \mathrm d d
                \int_{0}^{\theta} \frac{1}{2\pi} \delta \theta.
\end{align}
$$

Thus in the $(d, \theta)$ reference system the integral implicitly defining the joint density factors as the product of two analogous univariate integrals respectively depending on an exponential distribution of parameter $\lambda = \frac{1}{2}$ and on a uniform distribution over $[0, 2\pi)$. Therefore we can simulate these distributions in order to obtain values for $d$ and $\theta$ and subsequently map them onto the original $(X, Y)$ space. Summarizing, this amounts to generate two values $u_1, u_2$ uniformely distributed over $[0, 1]$ and subsequently computing

$$\begin{align}
\theta &= 2 \pi u_1, \\
d &= -2 \ln u_2, \\
x &= \sqrt{d} \cos \theta, \\
y &= \sqrt{d} \sin \theta.
\end{align}$$

The following panel implements and tests this simulation procedure. Note that each simulation round requires the generation of two uniform values and produces two normal specifications. In this case, we simply run five hundred simulation rounds and reshape the obtained results in order to get one thousand values as in previous cases. Note however that an implementation possibly returning one value should take into account the fact that specifications are produced in pairs, for instance exploiting a python generator.

```python
def std_random_bm():
    u1 = random.random()
    u2 = random.random()
    theta = 2 * math.pi * u1
    rho = math.sqrt(-2 * math.log(u2))
    return (rho * np.cos(theta), rho * np.sin(theta))

fig_bm_std_norm, ax_bm_std_norm = plt.subplots()

def new_bm_std_norm_sample(button):
    data = np.reshape([std_random_bm() for i in range(500)], 1000)

    ax_bm_std_norm.clear()

    x_ecdf, y_ecdf = ecdf(data)
    ax_bm_std_norm.plot(x_ecdf, y_ecdf, 'ko', markersize=2, alpha=.7)
    vals = np.arange(-3, 3, .1)
    ax_bm_std_norm.plot(vals, map(st.norm.cdf, vals))
    plt.xlim(-3, 3)

button_bm_std_normal = widgets.Button(description="Simulate")
button_bm_std_normal.on_click(new_bm_std_norm_sample)
new_bm_std_norm_sample(button_bm_std_normal)

display(button_bm_std_normal)
```

    <IPython.core.display.Javascript object>

### The approximate method ###

Despite its elegance, the Box-Muller method requires several multiplications and the evaluation of sine, cosine and logarithms, thus it is computationally expensive. The central limit theorem provides us with an approximate yet very fast method to simulate a standard normal variable. This theorem ensures that the sum of $n$ independent random variables with a common distribution $X$ approximately follows a normal variable having $n \mathrm E(X)$ as mean and $n \mathrm Var(X)$ as variance, and the approximation accuracy increases when $n$ grows. This theorem is typically exploited in order to obtain an approximate distribution of the sample mean $\overline X$ obtained from a sample $X_1, \dots, X_n$: as

$$\begin{align}
\mathrm E(\overline X) &= \mathrm E(X), \\
\mathrm{Var}(\overline X) &= \frac{\mathrm{Var}(X)}{n},
\end{align}$$

the random variable

$$
Z = \frac{\overline X - \mu}{\frac{sigma}{\sqrt{n}}}
$$

will approximately follow as a standard normal distribution. Now, if we rewrite $Z$ as

$$
Z = \frac{\frac{1}{n} \sum^{n}_{i=1}(X_i-\mu)} {\frac{\sigma}{\sqrt{n}}} =
\frac{1}{\sqrt n}\sum^{n}_{i=1}\frac{(X_i-\mu)}{\sigma} =
\frac{1}{\sqrt n}\sum^{n}_{i=1}X_i^*,
$$

where $X_i^*$ denotes the standardized version of $X_i$, an approximate simulation method for the standard normal distribution emerges:

- fix suitably $n \in \mathbb N$
- simulate independently $n$ specifications $x_1, \dots, x_n$ from a fixed distribution having $\mu$ and $\sigma$ respectively as mean and standard deviation
- standardize the specifications computing $x_i^* = \frac{x_i - \mu}{\sigma}$ for each $i = 1, \dots, n$
- compute and return the value $\frac{1}{\sqrt{n}} \sum_{i=1}^n x_i^*$

In order to implement this algorithm it is necessary to select the $X_i$'s distribution and to properly size $n$. Concerning the first degree of freedom, the simplest choice is that of relying on the uniform distribution over $[0, 1]$: recalling that if $U$ follows this distribution we have

$$\begin{align}
\mathrm E(U) &= \frac{1}{2}, \\
\mathrm{Var}(U) &= \frac{1}{12},
\end{align}$$

the proposed algorithm would consider $n$ random variables $U_1, \dots, U_n$ uniformly distributed over $[0, 1]$ and compute

$$
Z = \frac{1}{\sqrt{n}} \sum_{i=1}^n \frac{U_i - \frac{1}{2}}{\sqrt{\frac{1}{12}}}
= \sqrt{\frac{12}{n}} \left( \sum_{i=1}^n U_i -\frac{n}{2} \right).
$$

The obtained result suggests the choice $n=12$ leading to

$$
Z = \sum_{i=1}^{12} U_i - 6.
$$

```python
def std_random_appr():
    return sum([random.random() for i in range(12)]) - 6

fig_appr_std_norm, ax_appr_std_norm = plt.subplots()

def new_appr_std_norm_sample(button):
    data = [std_random_appr() for i in range(1000)]

    ax_appr_std_norm.clear()

    x_ecdf, y_ecdf = ecdf(data)
    ax_appr_std_norm.plot(x_ecdf, y_ecdf, 'ko', markersize=2, alpha=.7)
    vals = np.arange(-3, 3, .1)
    ax_appr_std_norm.plot(vals, map(st.norm.cdf, vals))
    plt.xlim(-3, 3)

button_appr_std_normal = widgets.Button(description="Simulate")
button_appr_std_normal.on_click(new_appr_std_norm_sample)
new_appr_std_norm_sample(button_appr_std_normal)

display(button_appr_std_normal)
```

    <IPython.core.display.Javascript object>

### Simulating a generic normal distribution ###

A generic normal distribution $N$ such that $\mathrm E(N) = \mu$ and $\mathrm{Var(N)} = \sigma^2$ can be easily simulated noting that $N$ has the same distribution of $\mu + \sigma Z$, where $Z$ is a standard normal variable.

```python
gen_norm_button = widgets.Button(description='Simulate')
gen_norm_mu_selector = widgets.FloatSlider(min=-5, max=5, value=2, description='$\\mu$')
gen_norm_sigma_selector = widgets.FloatSlider(min=.1, max=3, value=1, description='$\\sigma$')

fig_gen_norm_simulation, ax_gen_norm_simulation = plt.subplots()

def gen_normal(mu, sigma):
    z = std_random_appr()
    return mu + sigma * z

def gen_normal_simulation(mu, sigma):
    data = [gen_normal(mu, sigma) for i in range(1000)]
    vals = np.arange(mu - 3*sigma, mu + 3*sigma, .1)
    cdfs = map(lambda x: st.norm.cdf(x, loc=mu, scale=sigma), vals)

    clear_output()
    ax_gen_norm_simulation.clear()

    ax_gen_norm_simulation.plot(vals, cdfs, 'b')

    x_ecdf, y_ecdf = ecdf(data)
    ax_gen_norm_simulation.plot(x_ecdf, y_ecdf, 'ko', markersize=2, alpha=.7)
    plt.xlim(mu - 3*sigma, mu + 3*sigma)

    display(fig_gen_norm_simulation)

gen_norm_button.on_click(lambda b: gen_normal_simulation(gen_norm_mu_selector.value, gen_norm_sigma_selector.value))
display(gen_norm_button)

_ = interact(gen_normal_simulation, mu=gen_norm_mu_selector, sigma=gen_norm_sigma_selector)
```

    <IPython.core.display.Javascript object>

## Exercises ##

- Check that $n=12$ is sufficiently high for the approximation obtanied applying the central limit theorem to be meaningful.
- Implement a panel enhancing the test of the approximate method for generating a standard normal random variable adding a slider allowing to dynamically select the value for $n$.

```python

```


```python
import holoviews as hv
hv.extension('bokeh')
```

```python
import numpy as np
import scipy.signal
```

# Espectro y Transformada de Fourier

## Espectro de frecuencia

### Breve historia

- Isaac Newton llamó **espectros** a los componentes que forman la luz blanca y que normalmente no se pueden ver
- Newton mostró usando prismas que la luz blanca puede descomponerse en colores y viceverza
- Hoy entendemos que la luz como onda tiene una **frecuencia asociada** y que cada color es una frecuencia particular

Paradójicamente, Newton nunca acepto que esto se debía a la frecuencia de la radiación ya que creía en la teoría corpuscular de la luz

### Ondas y señales sinusoidales

Una onda es una perturbación que transporta energía a través del espacio

Tipicamente se describe por su frecuencia, amplitud y desfase. La frecuencia (Hertz) es el recíproco del período (segundos)

La siguiente es una función del tiempo completamente descrita por su amplitud $A$, frecuencia $f$ (período $P=1/f$) y fase $\phi$

$$
s(t) = A \cos (2 \pi f t + \phi)
$$

Observe en el siguiente ejemplo como cambia la señal al modificar sus parámetros

```python
def wave(time, frequency, phase, amplitude):
    return amplitude*np.cos(2.0*np.pi*time*frequency + phase)
```

```python
signal_plot = hv.HoloMap(kdims=['Amplitud', 'Frecuencia', 'Desfase'])

t = np.linspace(0, 2, num=1000)
for A in [0.5, 1.]:
    for f in [0.5, 1, 2]:
        for phi in np.arange(0., np.pi, np.pi/10):
            signal_plot[(A, f, phi)] = hv.Curve((t, wave(t, f, phi, A)))

signal_plot.opts(hv.opts.Curve(width=500, height=300))
```

### Componentes frecuenciales y descomposiciones armónicas

Una sinusoide que es períodica en una fracción

$$
\frac{P}{k} ~ \forall k \in \mathbb{N}
$$

también es periódica en $P$

En general llamamos a

- $f_0 = 1/P$ la **frecuencia fundamental**
- $f_k = k/P = kf_0$ el **k-esimo armónico de $f_0$**

Si sumamos armónicos con distintas amplitudes el resultado es una nueva función periódica que tiene la misma frecuencia fundamental

En el siguiente ejemplo modifique las amplitudes de cada armónico y observe la señal resultante

$$
s(t) = A_1 \cos(2\pi f_0 t ) + A_2 \cos(2\pi 2 f_0 t) + A_3 \cos(2\pi 3 f_0 t)
$$

Note que la suma de tres sinusoides no es necesariamente otra sinusoide

```python
t = np.linspace(0, 2, num=1000)

def my_signal(time, frequency, Amplitudes):
    s = 0
    for k, A in enumerate(Amplitudes):
        s += A*np.cos(2.0*np.pi*time*(k+1)*frequency)
    return s
```

```python
signal_plot = hv.HoloMap(kdims=['Amplitud fundamental',
                                'Amplitud 1er armónico',
                                'Amplitud 2do armónico',
                                'Frecuencia'])
for f in [1, 2]:
    for A1 in [0.5, 1.]:
        for A2 in [0, 0.5, 1.]:
            for A3 in [0, 0.5, 1.]:
                signal_plot[(A1, A2, A3, f)] = hv.Curve((t, my_signal(t, f, [A1, A2, A3])))

signal_plot.opts(hv.opts.Curve(width=500, height=300))
```

La noción de sintetizar señales en base a sinusoides se generaliza por medio de la **serie trigonométrica**

$$
\begin{align}
s(t) &= \sum_{k=0}^\infty A_k \cos(2\pi k f t + \phi_k) \nonumber \\
&= \sum_{k=0}^\infty a_k \cos(2\pi k f t) + b_k \sin(2\pi k f t), \nonumber
\end{align}
$$

donde $a_k = A_k \cos(\phi_k)$ y $b_k = -A_k \sin(\phi_k)$ se obtienen de $\cos(x+y) = \cos(x)\cos(y) - \sin(x)\sin(y)$

Con esto podemos generar funciones periódicas arbitrarias definiendo $\{a_k, b_k\}$ y $f$

**Ejemplo**

Sea por ejemplo $a_k = 0$ y $b_k = 1/k$  $\forall k$

¿Qué señal se obtiene al agregar cada vez más armónicos?

```python
t = np.linspace(0, 5, num=1000)

def trigonometric_series(time, frequency, K):
    s = 0
    for k in range(1, K):
        s += np.sin(2.0*np.pi*time*k*frequency)/(k)
    return s
```

```python
signal_plot = hv.HoloMap(kdims=['Cantidad de armónicos', 'Frecuencia'])
for f in [1, 2]:
    for K in [1, 2, 3, 5, 10, 20, 50, 100]:
        signal_plot[(K, f)] = hv.Curve((t, trigonometric_series(t, f, K)))

signal_plot.opts(hv.opts.Curve(width=500, height=300))
```

```{note}
Hasta ahora hemos visto como sintetizar señales a partir de sus armónicos, pero también podemos hacer el proceso inverso, es decir encontrar los armónicos de una señal dada
```

Tal como Newton descompuso la luz blanca en colores, nosotros podemos descomponer una señal en sus armónicos usando la **serie de Fourier**

## Serie de Fourier

Un poco de historia: En 1807 *Jean Baptiste Joseph Fourier* presenta un teorema indicando que una función periódica arbitraria con periódo $P=1/f_0$ puede representarse como una suma ponderada de senos y cosenos

La serie de Fourier (FS) es una generalización de la serie trigonométrica a los números complejos. Puede revisar el apéndice de esta lección si necesita refrescar la memoria sobre los números complejos y sus representaciones

La FS se define como

$$
s(t) = \sum_{k=-\infty}^{\infty} c_k e^{j 2\pi k f_0 t}, ~~ c_k \in \mathbb{C}
$$

A continuación veremos como obtener los coeficientes de Fourier $c_k$ para una señal periódicas arbitraria

### Base de Fourier

El conjunto de funciones

$$
v_k (t) = \frac{1}{\sqrt{P}} e^{j2\pi k t / P} ~~ \forall k \in \mathbb{Z}
$$

se conoce como base de Fourier y cumple con

$$
\langle v_n (t), v_m (t) \rangle = \int_0^P v_n (t) v_m^* (t) dt = \frac{1}{P} \int_0^P e^{j2\pi (n-m)t/P} dt =\begin{cases}1 & n=m \\ 0 & n \neq m\end{cases}
$$

La base de Fourier es un conjunto ortonormal en el espacio de funciones periódicas con periódo $P$. Esto es facilmente comprobable si estudiamos

$$
\int_0^P e^{j2\pi k t / P} dt = \int_0^P \cos(2\pi k t/P) dt + j \int_0^P \sin(2\pi k t/P) dt
$$

Podemos usar esta propiedad y escribir el producto punto entre una señal y el m-esimo elemento de la base de Fourier

$$
\begin{align}
\langle s(t),  e^{j 2\pi m t/ P} \rangle &= \int_0^P s(t) e^{-j 2\pi m t/ P} dt \nonumber \\
&= \int_0^P \sum_{k=-\infty}^{\infty} c_k e^{j 2\pi k t/P} e^{-j 2\pi m t/P} dt \nonumber \\
&= \sum_{k=-\infty}^{\infty} c_k  \int_0^P e^{j 2\pi (k-m) t/P}  dt \nonumber \\
&=  c_m  P \nonumber \\
\end{align}
$$

Esto nos da una forma sencilla para obtener los $c_m$ de una señal periódica arbitraria

$$
c_m = \frac{1}{P} \int_0^P s(t) e^{-j 2\pi m t/P} dt
$$

### Ejemplo formativo: FS de señal cuadrada

Sea

$$
s(t) = \begin{cases} 1 & t \in[0, \frac{P}{2}] \\ 0 & t \in [\frac{P}{2}, P] \end{cases}
$$

Los coeficientes de su FS son

$$
c_0 = \frac{1}{P} \int_0^P s(t) dt = \frac{1}{P} \int_0^{P/2} dt = \frac{1}{2}
$$

y

$$
\begin{align}
c_k &= \frac{1}{P} \int_0^\frac{P}{2} e^{-j2\pi kt/P} dt  \nonumber  \\
&= -\frac{j}{P}  \int_0^\frac{P}{2} \sin(2\pi kt/P) dt \nonumber \\
&=  0 + j \frac{\cos(\pi k)  - 1}{2\pi k} \nonumber
\end{align}
$$

Notemos que a excepción de $c_0$ los coeficientes sólo tienen parte imaginaria. Además sólo los armónicos impares (senos) son distintos de cero

Finalmente la FS está dada por

$$
\begin{align}
s(t) &= \sum_{k=-\infty}^{\infty} j \frac{\cos(\pi k)  - 1}{2\pi k}  e^{j 2\pi k t/P} \nonumber \\
&= \frac{1}{2} + \sum_{k=1}^{\infty}  \frac{1 - \cos(\pi k)}{\pi k} \sin(2\pi k t/P)  \nonumber
\end{align}
$$

Programemos esta expresión y visualicemos el resultado al incrementar cada vez la cantidad de armónicos

```python
t = np.linspace(0, 5, num=1000)

def FS_square(tiempo, frecuencia, K):
    s = 0.5 # C_0
    for k in range(1, K):
        c_k = (1-np.cos(np.pi*k))/(np.pi*k)
        s += c_k*np.sin(2.0*np.pi*tiempo*k*frecuencia)
    return s
```

```python
signal_plot = hv.HoloMap(kdims=['Cantidad de armónicos', 'Frecuencia', ])

for f in [1, 2]:
    for K in [1, 2, 3, 5, 10, 20, 50, 100]:
        signal_plot[(K, f)] = hv.Curve((t, FS_square(t, f, K)))

signal_plot.opts(hv.opts.Curve(width=500, height=300))
```

### Propiedades de la FS

A continuación se enumeran algunas de las propiedades más importantes de la FS

- Si $s(t)$ es par entonces $c_k$ es par
- Si $s(t)$ es impar entonces $c_k$ es impar
- Si $s(t + P/2) = -s(t)$ (antiperiódica) entonces $c_k=0$ para k par
- Si $s(t)$ es real y par entonces $c_k$ es real y par
- Si $s(t)$ es real e impar entonces $c_k$ es imaginario e impar
- **La FS es lineal**

El **Teorema de Parseval** relaciona la potencia de una señal periódica arbitraria con sus coeficientes de Fourier

$$
\begin{align}
P_s &= \frac{1}{P} \int_0^P |s(t)|^2 dt \nonumber \\
&= \frac{1}{P} \int_0^P |\sum_{k=-\infty}^{\infty} c_k e^{j 2\pi k t/P }|^2 dt \nonumber \\
&= \frac{1}{P} \int_0^P \sum_{k=-\infty}^{\infty} |c_k |^2 dt \nonumber \\
&= \sum_{k=-\infty}^\infty |c_k|^2 \nonumber
\end{align}
$$

## Transformada de Fourier

Podemos extender el concepto de descomposición armónica a señales no periódicas usando la **Transformada de Fourier**. Esta herramienta matemática será fundamental en este curso

- El concepto de frecuencia puede aplicarse también a señales no-periódicas
- Según **Joseph Fourier** una señal no-periódica puede ser vista como una señal periódica **con un período infinito**
- El único requisito es que ahora las frecuencias son un continuo, con un espaciado infinitesimal

A continuación veremos que una señal analógica puede ser vista como continua en el tiempo o continua en frecuencia. En la próxima lección veremos la Transformada de Fourier discreta (DFT) para señales digitales

### Derivación de la transformada de Fourier a partir de la FS

Sea un tren de pulsos cuadrado con periódo P y ancho $2T < P$ definido en un período como

$$
s(t) = \begin{cases} 1, & |t| < T \\ 0, & T<|t| < P/2 \end{cases}
$$

Los coeficientes de su serie de Fourier son

$$
c_0 = \frac{1}{P} \int_{-P/2}^{P/2} s(t) dt = \frac{1}{P} \int_{-T}^{T}  dt = \frac{2T}{P}
$$

y

$$
\begin{align}
c_k &= \frac{1}{P} \int_{-T}^{T} e^{-j2\pi kt/P} dt  \nonumber \\
&= \frac{1}{\pi k} \sin \left (2\pi k \frac{T}{P} \right)  \nonumber
\end{align}
$$

En el siguiente ejemplo la figura izquierda muestra la señal de pulso cuadrado mientras que la derecha muestra $c_k$ en función de $k$

¿Qué ocurre con $c_k$ a medida que $P$ aumenta ($f$ disminuye)?

```python
def fs_coef(P, T, maxK=200):
    K = np.arange(-maxK, maxK)
    ck = np.zeros_like(K, dtype='float32')
    ck[len(K)//2] = 2*T/P
    for k in range(1, K[-1]):
        ck[len(K)//2+k] = np.sin(2*np.pi*k*T/P)/(np.pi*k)
        ck[len(K)//2-k] = ck[len(K)//2+k]
    return K, ck

def synthesis(time, ck, P):
    maxK = len(ck)//2
    s = ck[maxK]
    for k in range(1, maxK):
        s += 2*np.cos(2*np.pi*time*k/P)*ck[maxK+k]
    return s
```

```python
t = np.linspace(-3, 3, num=2000)
T = 0.25

signal_plot = hv.HoloMap(kdims=['Periodo'])
fs_plot = hv.HoloMap(kdims=['Periodo'])
for P in [1, 2, 3, 5, 10]:
    K, ck = fs_coef(P, T)
    fs_plot[P] =  hv.Spikes((K, P*ck), 'Frecuencia (k)', 'P x ck').opts(xlim=(-50, 50))
    signal_plot[P] =  hv.Curve((t, synthesis(t, ck, P)), 'Tiempo (segundos)', 'Señal')

(signal_plot+fs_plot)
```

De la figura tenemos que cuando P es grande (o equivalentemente f es pequeño)

- el tren de pulsos tiende a un único pulso
- los coeficientes toman una "forma suave"

Llamaremos a esta forma suave de $P \cdot c_k$ la "envolvente" de los coeficientes. Si tomamos el caso $P \to \infty$ la envolvente resultante es

$$
\lim_{P\to \infty} P c_k = S(f) = \int_{-\infty}^{\infty} s(t) e^{-j 2\pi t  f} dt,
$$

o también

$$
S(\omega) = \int_{-\infty}^{\infty} s(t) e^{-j\omega t } dt,
$$

donde $\omega = 2\pi f$ se llama frecuencia angular.

```{note}
Esto se conoce como transformada de Fourier directa o integral de Fourier
```

Reemplazando $c_k = f_0 S(k f_0)$ y tomando el límite cuando $f_0 = \frac{1}{P} \to 0$ tenemos

$$
\begin{align}
s(t) &= \lim_{f_0 \to 0} \sum_{k=-\infty}^{\infty} f_0 S(k f_0) e^{j 2\pi t  k f_0} \nonumber  \\
&= \int_{-\infty}^{\infty} S(f) e^{j 2\pi t  f} df \nonumber
\end{align}
$$

o también

$$
s(t) = \frac{1}{2\pi} \int_{-\infty}^{\infty} S(\omega) e^{j \omega t } d\omega,
$$

que se conoce como la transformada de Fourier inversa

### Par de Fourier

Los operadores (transformada de Fourier directa)

$$
S(\omega) = \mathbb{FT} [s(t)] = \int_{-\infty}^{\infty} s(t) e^{-j\omega t } dt
$$

y (transformada de Fourier inversa)

$$
s(t) = \mathbb{FT}^{-1} [S(\omega)] = \frac{1}{2\pi} \int_{-\infty}^{\infty} S(\omega) e^{j \omega t } d\omega, \nonumber
$$

se conocen como par de Fourier y nos permiten analizar una señal en el dominio del tiempo o en el dominio de la frecuencia sin pérdidas

### Ejemplo: Transformada del pulso cuadrado

Consideremos nuevamente el pulso cuadrado

$$
s(t) = \begin{cases} 1, & |t| < T \\ 0, & |t| > T\end{cases}
$$

su transformada de Fourier es

$$
\begin{align}
S(\omega) &= \int_{-\infty}^{\infty} s(t) e^{-j\omega t } dt \nonumber \\
&= \int_{-T}^{T} e^{-j\omega t } dt \nonumber \\
&= \frac{1}{-j\omega} \left(e^{-j\omega T }  - e^{j\omega T } \right) \nonumber \\
&= \frac{2}{\omega}  \sin(\omega T) = 2T \text{sinc}(\omega T) \nonumber
\end{align}
$$

Esto calza precisamente con la "envolvente" que vimos anteriomente

```python
T = 0.25

w = np.arange(-100, 100)
S = 2*np.sin(w*T)/w
S[w==0] = 0.5
```

```python
hv.Curve((w, S), 'Frecuencia angular', 'Espectro').opts(width=500)
```

### Propiedades de la transformada de Fourier

1) La transformada de Fourier es un operador lineal, si tenemos dos señales y dos valores escalares entonces

$$
\mathbb{FT}[c_1 s_1(t) + c_2 s_2(t)] = c_1\mathbb{FT}[s_1(t)] + c_2\mathbb{FT}[s_2(t)]
$$

2) El operador de convolución en el tiempo se convierte en multiplicación en frecuencia

$$
\mathbb{FT}[(s_1 * s_2)(t)] =  \mathbb{FT}[s_1(t)] \cdot \mathbb{FT}[s_2(t)],
$$

donde

$$
(s_1 * s_2)(t) = \int s_1(\tau) s_2(t-\tau) d\tau
$$

es la operación de convolución

3) Así mismo, la multiplicación en frecuencia se convierte en multiplicación en el tiempo

$$
\mathbb{FT}[s_1(t)\cdot s_2(t)] =  \frac{1}{2\pi}\mathbb{FT}[s_1(t)] * \mathbb{FT}[s_2(t)]
$$

Finalmente el Teorema de Parseval nos dice que la energía se preserva entre frecuencia y tiempo

$$
\int | s(t) |^2 dt = \frac{1}{2\pi} \int | S(\omega) |^2 d\omega
$$

### Definición: Espectros de amplitud y fase

Llamamos a $S(\omega)$ el espectro (transformada de Fourier) de una señal $s(t)$

El espectro es un número complejo que podemos escribir en notación polar como

$$
S(\omega) = |S(\omega)| e^{j\Phi(\omega)},
$$

donde $|S(\omega)|$ se conoce como **espectro de amplitud** y $\Phi(\omega)$ como **espectro de fase**

## Resumen de la lección

En esta lección aprendimos

- a realizar síntesis de señales usando sinusoides
- a descomponer señales periódicas usando la serie de Fourier
- a descomponer señales periódicas y no periódicas usando la transformada de Fourier

## Apéndice: Números complejos

Sea z un número complejo, lo podemos escribir en forma cartesiana

$$
z = \Re[z] + j \Im[z] = a + j b
$$

donde $a \in \mathbb{R}$, $b \in \mathbb{R}$ y $j = \sqrt{-1}$ es el número imaginario.

También podemos escribirlo en forma polar

$$
z = c e^{j\phi} = c \cos(\phi) + j c \sin(\phi)
$$

donde
- $c = |z| = \sqrt{a^2 + b^2} \in [0, \infty]$ es la magnitud
- $\phi = \angle z = \tan^{-1} \left (\frac{b}{a} \right) \in [-\frac{\pi}{2}, \frac{\pi}{2}]$ es el ángulo
- $a = c \cos(\phi)$
- $b = c\sin(\phi)$

se pueden escribir las siguientes relaciones

$$
\cos(\phi) = \frac{1}{2} (e^{j\phi} + e^{-j\phi}) ~\wedge~ \sin(\phi) = \frac{1}{2j} (e^{j\phi} - e^{-j\phi})
$$

el complejo conjugado de $z = a + j b = c e^{j\phi}$ es

$$
z^* = a - jb = c e^{-j\phi}
$$

```python

```


# Defining custom problems
Many classic optimisation problems are [already implemented in PYMOO](https://www.pymoo.org/problems/index.html). However, you will want to implement your own problem at some point. Here's how to do that.

```python
import numpy as np
import autograd.numpy as anp
from pymoo.model.problem import Problem
from pymoo.util.misc import stack
from pymoo.model.problem import FunctionalProblem
```

Let's consider the following problem.

\begin{align}
\min \quad f_1(x) & = (x_1^2 + x_2^2) \\
\min \quad f_2(x) & = (x_1 - 1)^2 + x_2^2 \\
\textrm{s.t.} \quad
g_1(x) & = 2(x_1−0.1)(x_1−0.9)/0.18 ≤ 0 \\
g_2(x) & = −20(x_1−0.4)(x_1−0.6)/4.8 ≤ 0 \\
−2 & ≤ x_1 ≤ 2  \\
−2 & ≤ x_2 ≤ 2
\end{align}

It consists of:
- 2 objective functions $f_1(x)$ and $f_2(x)$
- 2 design variables $x_1$ and $x_2$, both in the range $[−2,2]$
- 2 inequality contraints $g_1(x)$ and $g_2(x)$

There are 3 methods to define a custom problem in PYMOO: vectorized problem, elementwise problem and functional problem.

## Vectorized problem
The _evaluate method takes a **two-dimensional** NumPy array X with n rows and m columns as an input. Each row represents an individual, and each column an optimization variable. After doing the necessary calculations, the objective values must be added to the dictionary out with the key F and the constraints with key G.

```python
class MyVectorizedProblem(Problem):

    def __init__(self):
        super().__init__(n_var=2,
                         n_obj=2,
                         n_constr=2,
                         xl=np.array([-2,-2]),
                         xu=np.array([2,2]))

    def _evaluate(self, X, out, *args, **kwargs):
        f1 = X[:, 0]**2 + X[:, 1]**2
        f2 = (X[:, 0]-1)**2 + X[:, 1]**2

        g1 = 2*(X[:, 0]-0.1) * (X[:, 0]-0.9) / 0.18
        g2 = - 20*(X[:, 0]-0.4) * (X[:, 0]-0.6) / 4.8

        out["F"] = np.column_stack([f1, f2])
        out["G"] = np.column_stack([g1, g2])

vectorized_problem = MyVectorizedProblem()
```

## Elementwise problem
The _evaluate method takes a **one-dimensional** NumPy array x number of entries equal to n_var. This behavior is enabled by setting elementwise_evaluation=True while calling the super() method.

```python
class MyElementwiseProblem(Problem):

    def __init__(self):
        super().__init__(n_var=2,
                         n_obj=2,
                         n_constr=2,
                         xl=np.array([-2,-2]),
                         xu=np.array([2,2]),
                         elementwise_evaluation=True)

    def _evaluate(self, x, out, *args, **kwargs):
        f1 = x[0]**2 + x[1]**2
        f2 = (x[0]-1)**2 + x[1]**2

        g1 = 2*(x[0]-0.1) * (x[0]-0.9) / 0.18
        g2 = - 20*(x[0]-0.4) * (x[0]-0.6) / 4.8

        out["F"] = [f1, f2]
        out["G"] = [g1, g2]

elementwise_problem = MyElementwiseProblem()
```

## Functional problem

```python
objs = [
    lambda x: x[0]**2 + x[1]**2,
    lambda x: (x[0]-1)**2 + x[1]**2
]

constr_ieq = [
    lambda x: 2*(x[0]-0.1) * (x[0]-0.9) / 0.18,
    lambda x: - 20*(x[0]-0.4) * (x[0]-0.6) / 4.8
]

functional_problem = FunctionalProblem(n_var=2,
                                       objs=objs,
                                       constr_ieq=constr_ieq,
                                       xl=np.array([-2,-2]),
                                       xu=np.array([2,2]))
```

## More vectorized problem examples

```python
class SphereWithConstraint(Problem):

    def __init__(self):
        super().__init__(n_var=10,
                         n_obj=1,
                         n_constr=1,
                         xl=0,
                         xu=1)

    def _evaluate(self, x, out, *args, **kwargs):
        out["F"] = np.sum((x - 0.5) ** 2, axis=1)
        out["G"] = 0.1 - out["F"]

sphere_problem = SphereWithConstraint()
```

```python
class MyVectorizedProblem2(Problem):

    def __init__(self, const_1=5, const_2=0.1):

        # define lower and upper bounds -  1d array with length equal to number of variable
        xl = -5 * anp.ones(10)
        xu = 5 * anp.ones(10)

        super().__init__(n_var=10,
                         n_obj=1,
                         n_constr=2,
                         xl=xl,
                         xu=xu,
                         evaluation_of="auto")

        # store custom variables needed for evaluation
        self.const_1 = const_1
        self.const_2 = const_2

    def _evaluate(self, x, out, *args, **kwargs):
        f = anp.sum(anp.power(x, 2) - self.const_1 * anp.cos(2 * anp.pi * x), axis=1)
        g1 = (x[:, 0] + x[:, 1]) - self.const_2
        g2 = self.const_2 - (x[:, 2] + x[:, 3])

        out["F"] = f
        out["G"] = anp.column_stack([g1, g2])

vectorized_problem_2 = MyVectorizedProblem2()
```

## Another elementwise problem example

```python
class MyElementwiseProblem2(Problem):

    def __init__(self, **kwargs):
        super().__init__(n_var=2,
                         n_obj=1,
                         elementwise_evaluation=True,
                         **kwargs)

    def _evaluate(self, x, out, *args, **kwargs):
        out["F"] = x.sum()

elementwise_problem_2 = MyElementwiseProblem2()
```

## Another functional problem example

```python
objs = [
    lambda x: np.sum((x - 2) ** 2),
    lambda x: np.sum((x + 2) ** 2)
]

constr_ieq = [
    lambda x: np.sum((x - 1) ** 2)
]

functional_problem_2 = FunctionalProblem(10,
                            objs,
                            constr_ieq=constr_ieq,
                            xl=np.array([-10, -5, -10]),
                            xu=np.array([10, 5, 10])
                            )
```

# Problem evaluation
We can evaluate a problem with the evaluate() method. This method takes as input a two dimensional matrix where each row is a point to evaluate and each column a variable and returns a dictionary with the result of the evaluation.

```python
# input vector: a random population of 3 individuals, each defined by 2 or 10 design variables.
X_2 = np.random.rand(3, 2)
X_10 = np.random.rand(3, 10)
```

```python
vectorized_problem.evaluate(X_2, return_as_dictionary=True)
```

    {'F': array([[0.08865958, 0.92178874],
            [0.25352178, 0.94993085],
            [0.06625316, 1.06159225]]), 'CV': array([[0.15028942],
            [0.        ],
            [0.97416637]]), 'G': array([[ 0.15028942, -0.68135853],
            [-0.4305956 , -0.46352665],
            [ 0.97416637, -0.99031239]])}

```python
elementwise_problem.evaluate(X_2, return_as_dictionary=True)
```

    {'F': array([[0.08865958, 0.92178874],
            [0.25352178, 0.94993085],
            [0.06625316, 1.06159225]]), 'CV': array([[0.15028942],
            [0.        ],
            [0.97416637]]), 'G': array([[ 0.15028942, -0.68135853],
            [-0.4305956 , -0.46352665],
            [ 0.97416637, -0.99031239]])}

```python
functional_problem.evaluate(X_2, return_as_dictionary=True)
```

    {'F': array([[0.08865958, 0.92178874],
            [0.25352178, 0.94993085],
            [0.06625316, 1.06159225]]), 'CV': array([[0.15028942],
            [0.        ],
            [0.97416637]]), 'G': array([[0.15028942, 0.        ],
            [0.        , 0.        ],
            [0.97416637, 0.        ]])}

```python
sphere_problem.evaluate(X_10, return_as_dictionary=True)
```

    {'F': array([[0.75168175],
            [0.83946771],
            [1.20842157]]), 'CV': array([[0.],
            [0.],
            [0.]]), 'G': array([[-0.65168175],
            [-0.73946771],
            [-1.10842157]])}

```python
# We can return more values in the output dictionary
vectorized_problem_2.evaluate(X_10,
                              return_as_dictionary=True,
                              return_values_of=["F", "G", "CV", "feasible", "dF", "dG"])
```

    {'F': array([[ 13.02018527],
            [  2.152319  ],
            [-17.00788227]]), 'G': array([[ 0.90149626, -1.01947164],
            [ 0.82882442, -0.52283495],
            [ 1.57541812, -0.37100353]]), 'CV': array([[0.90149626],
            [0.82882442],
            [1.57541812]]), 'feasible': array([[False],
            [False],
            [False]]), 'dF': array([[[  3.30302764,  -1.59446704,  -3.98945639, -16.27825395,
               30.18928505,   5.39321748, -19.75971148, -11.25048504,
                1.60219873, -27.99258105]],

            [[ -2.64141317,  17.62966488,  31.89409186,  22.63569441,
              -21.58133575,   5.47558745, -23.60753848, -18.56252313,
               22.96358009, -28.8773237 ]],

            [[ -1.88687626, -28.16606694,  31.67896801,  30.26315304,
               22.28974347,  30.88947072,  -7.9440922 ,  15.36147014,
               31.89275415,  15.38885776]]]), 'dG': array([[[ 1.,  1.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.],
             [ 0.,  0., -1., -1.,  0.,  0.,  0.,  0.,  0.,  0.]],

            [[ 1.,  1.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.],
             [ 0.,  0., -1., -1.,  0.,  0.,  0.,  0.,  0.,  0.]],

            [[ 1.,  1.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.],
             [ 0.,  0., -1., -1.,  0.,  0.,  0.,  0.,  0.,  0.]]]), '__autograd__': array([[0.48820239, 0.51329387, 0.52564761, 0.59382403, 0.19860487,
             0.02717889, 0.87990381, 0.56446297, 0.0080388 , 0.69234881],
            [0.51867974, 0.41014468, 0.24546344, 0.37737151, 0.86690449,
             0.02759812, 0.64562427, 0.88793008, 0.12856541, 0.7894407 ],
            [0.98045779, 0.69496033, 0.27124377, 0.19975976, 0.12377091,
             0.29245994, 0.94928248, 0.08038163, 0.25793561, 0.0805384 ]])}

```python
# in this problem, there is no constraints defined. The single objective function returns the sum of the 2 design variables.
# elementwise_problem_2.evaluate(X_2, return_as_dictionary=True)
elementwise_problem_2.evaluate([1, 0.5], return_as_dictionary=True)
```

    {'F': array([1.5])}

```python
functional_problem_2.evaluate(X_10, return_as_dictionary=True)
```

    {'F': array([[24.77716362, 60.70921197],
            [23.64630035, 62.82807997],
            [26.91604974, 58.36237461]]), 'CV': array([[3.76017571],
            [3.44174526],
            [4.77763096]]), 'G': array([[3.76017571],
            [3.44174526],
            [4.77763096]])}

```python

```

```python

```


# Normal Distribution

> ***GitHub***: https://github.com/czs108

## Definition

The *Normal Distribution* is in the shape of a bell curve. The curve is *symmetrical*, with the *highest* probability density in the *center* of the curve. The probability density *decreases* when getting away from the *mean*. Both the *mean* and *median* are at the center and have the highest probability density.

The *Normal Distribution* is defined by **2** parameters. $\mu$ is where the center of the curve is, and $\sigma^{2}$ is the spread. If a continuous random variable $X$ follows a *Normal Distribution* with *mean* $\mu$ and *standard deviation* $\sigma$, this is generally written as:

\begin{equation}
X \sim N(\mu,\, \sigma^{2})
\end{equation}

As $\sigma^{2}$ gets *larger*, the *flatter* and *wider* the normal curve becomes.

## Expectation

\begin{equation}
E(X) = \mu
\end{equation}

## Variance

\begin{equation}
Var(X) = \sigma^{2}
\end{equation}

## Calculating

### Standardization

\begin{equation}
Z = \frac{X - \mu}{\sigma}
\end{equation}

### Probability Table

Start off by calculating $z$ to **2** decimal places, then read off the corresponding probability $P(Z < z)$.

The *1st* *column* gives the value of $z$ to **1** decimal place (without rounding), and the top *row* gives the *2nd* decimal place. The probability is where the two intersect.

| z       | .00    | .01    | .02    | .03    | .04    | ...  |
| ------- | ------ | ------ | ------ | ------ | ------ | ---- |
| **0.0** | 0.5000 | 0.5040 | 0.5080 | 0.5120 | 0.5160 | ...  |
| **0.1** | 0.5398 | 0.5438 | 0.5478 | 0.5517 | 0.5557 | ...  |
| ...     | ...    | ...    | ...    | ...    | ...    | ...  |

For example, if you wanted to find $P(Z < 0.12)$, you'd find **0.1** in the *1st* column, **.02** in the top row, and read off a probability of **0.5478**.

## Approximation

Suppose $q$ is $(1 - p)$, a *Binomial Distribution* $X \sim B(n,\, p)$ can be approximated by $X \sim N(np,\, npq)$ normally if $np > 5$ and $nq > 5$.


# Project 2

Consider the truss assembly pictured in the figure below.

Your job as an engineer is to choose the structural steel required to ensure the truss assembly shown in the figure will not fail under a variety of loading conditions.  The external forces applied to the truss are always applied at the locations $B$ and $D$ in the directions shown.  The external forces can be summarized as

\begin{equation}
\vec{F}=\left[
\begin{array}{c}
{B}_y \\
{D}_x \\
{D}_y
\end{array}
\right]
\end{equation}

In order to determine which structural steel to use we must first determine the force in each truss member under a variety of loading conditions.  A survey of the applications for this truss has resulted in the following possible loading conditions

\begin{equation}
\vec{F}_1=\left[
\begin{array}{c}
400 \mathrm{N} \\
600 \mathrm{N} \\
0 \mathrm{N}
\end{array}
\right],
%
\vec{F}_2=\left[
\begin{array}{c}
700 \mathrm{N} \\
100 \mathrm{N} \\
200 \mathrm{N}
\end{array}
\right],
%
\vec{F}_3=\left[
\begin{array}{c}
100 \mathrm{N} \\
1000 \mathrm{N} \\
100 \mathrm{N}
\end{array}
\right],
%
\vec{F}_4=\left[
\begin{array}{c}
0 \mathrm{N} \\
500 \mathrm{N} \\
500 \mathrm{N}
\end{array}
\right],
%
\vec{F}_5=\left[
\begin{array}{c}
500 \mathrm{N} \\
0 \mathrm{N} \\
1500 \mathrm{N}
\end{array}
\right],
\end{equation}
\begin{equation}
%
\vec{F}_6=\left[
\begin{array}{c}
1200 \mathrm{N} \\
0 \mathrm{N} \\
100 \mathrm{N}
\end{array}
\right],
%
\vec{F}_7=\left[
\begin{array}{c}
1600 \mathrm{N} \\
0 \mathrm{N} \\
0 \mathrm{N}
\end{array}
\right],
%
\vec{F}_8=\left[
\begin{array}{c}
0 \mathrm{N} \\
0 \mathrm{N} \\
1600 \mathrm{N}
\end{array}
\right],
%
\vec{F}_9=\left[
\begin{array}{c}
0 \mathrm{N} \\
1600 \mathrm{N} \\
0 \mathrm{N}
\end{array}
\right],
%
\vec{F}_{10}=\left[
\begin{array}{c}
500 \mathrm{N} \\
500 \mathrm{N} \\
500 \mbox{N}
\end{array}
\right].
\end{equation}

Using either Gaussian elimination or LU Decomposition (you can use your solution from [assignment13](https://github.com/PGE310-Students/assignment13) and/or [assignment14](https://github.com/PGE310-Students/assignment14), or the built in NumPy/SciPy implementations) complete the function `compute_largest_member_force` below to efficiently solve for the forces in all members of the truss.  Using this information determine what the maximum load (compressive or tensile) is in any member for each loading configuration, the largest force (in magnitude) in any member, and which member contains the largest force.

The function `compute_largest_member_force` should take a two-dimensional NumPy array as the input argument.  Each row of the input array corresponds to one of the loading conditions $F_1, F_2, \dots, F_{10}$.  So the array should be a 3 by 10 for these particular loads.

The function should return a three-tuple that contains `(member_id_string, force_id_string, magnitude_largest_force)` where `member_id_string` is a string that labels the member that contains the largest force, so one of `'AD', 'AB', 'BD', 'CD',` or `'BC'`.  `force_id_string` is a string that labels the loading condition that gives rise to the maximum force so one of `'F1', 'F2', 'F3', 'F4', 'F5', 'F6', 'F7', 'F8', 'F9',` or `'F10'`. `magnitude_largest_force` is the magnitude of the largest force in any member.  So an example of what the function should return is `('AB', 'F1', 102.5)` in that order!

If you use your assignment13 or 14 codes and import them after converting to Python, make sure you add those files to the repository when you commit them.

**Hint:** You must first solve for the reaction at the pins of the entire truss in equilibrium.  These reaction forces along with the externally applied loads can then be used with the Method of Sections or the Method of Joints to write the equilibrium equations for each member of the truss.

```python
def compute_largest_member_force(forces):

    return #(member_id_string, force_id_string, magnitude_largest_force)
```


# Uncertainty of efficiency  from fitted decay yields

We derive an approximate formula to compute the uncertainty of the efficiency computed from fitted yields of some decays. The formula is suitable to draw error bars in plots.

```python
#!pip install numpy iminuit numba_stats matplotlib scipy sympy joblib
import numpy as np
from iminuit import Minuit
from iminuit.cost import ExtendedBinnedNLL, NormalConstraint
from numba_stats import bernstein, norm
from matplotlib import pyplot as plt
import joblib
from sympy import *
```

## Introduction

The efficiency of a selection is

$$
\epsilon = \frac{k}{n},
$$

where $k$ is the number of events which pass and $n$ is the total number of events.

For the sake of computing confidence intervals for an efficiency computed from a sample, $\epsilon$ is typically assumed to be a [binomial proportion](https://en.wikipedia.org/wiki/Binomial_proportion_confidence_interval). This means that $k$ is binomially distributed for fixed $n$ in repeated experiments.

In practice, this assumption is usually incorrect. The efficiency is usually computed from a total sample of events whose number $n$ is Poisson distributed, since experiments that involve random processes are typically run for a fixed time rather than until a fixed number of events has occured. This produces a larger variance for $\epsilon$ compared to what is expected from the binomial model.

We demonstrate this numerically.

```python
rng = np.random.default_rng(1)

x = np.linspace(0, 5, 100)

# sigmoid
p = 1.0 / (1.0 + np.exp(-x))

plt.plot(x, p, color="k", label=r"$\epsilon(x) = 1/(1 + e^{-x})$")

ntry = 10000
for i, n in enumerate((5, 10, 100)):
    eff1 = []
    eff2 = []
    for pi in p:
        y1 = rng.binomial(n, pi, size=ntry) / n

        ni = rng.poisson(n, size=ntry)
        with np.errstate(invalid="ignore"):
            y2 = rng.binomial(ni, pi) / ni

        for y, eff in [(y1, eff1), (y2, eff2)]:
#             q = np.nanquantile(y, [0.16, 1-0.16])
            m = np.nanmean(y)
            s = np.nanstd(y)
            eff.append((m - s, m + s))
    eff1 = np.array(eff1)
    eff2 = np.array(eff2)

    col = f"C{i}"
    for eff, ls, label in ((eff1, ":", f"n={n} fixed"),
                           (eff2, "--", f"poisson n with $\\langle n \\rangle = {n}$ ")):
        plt.plot(x, eff[:, 0], color=col, ls=ls, label=label)
        plt.plot(x, eff[:, 1], color=col, ls=ls)

    print(np.mean(np.diff(eff2, axis=1) / np.diff(eff1, axis=1)))

plt.legend()
plt.xlabel(r"$x$")
plt.ylabel(r"$\epsilon$");
```

Shown is an sigmoidal efficiency curve and a normal approximation interval computed from the standard deviation of the sample estimate $\hat \epsilon$. For small $n$, the interval is wider if $n$ is Poisson distributed.

We compute the variance of $\epsilon$ for both cases explicitly. In the following, it will be important to clearly distinguish sample values from their population values. We will use $\hat x$ for the sample value, which has a population value $x$. The sample value of the efficiency then is $\hat \epsilon = \hat k / \hat n$.

## No background, n fixed

If $\hat n = n$ is fixed, the variance of $\hat \epsilon$ can be computed from the variance of the binomial distribution $\text{var}(\hat k) = n \epsilon (1 - \epsilon)$. We expand the formula for $\epsilon$ in a Taylor series in the variable $k = \epsilon n$ around the sample value $\hat k$ and obtain

$$
\epsilon = \sum_{i=0}^{\infty} \frac {(k - \hat k) ^i}{i!} \frac{\partial^i \epsilon}{\partial {k}^i}(\hat k) = \hat \epsilon + \frac {k - \hat k} n.
$$

The series naturally break off after the first order. Thanks to that, we can compute the variance of $\hat \epsilon$ exactly,

$$
\text{var}(\hat\epsilon) = \text{E}\big((\hat \epsilon - \epsilon)^2\big) = \text{E}\left(\frac{(\hat k - k)^2}{n^2}\right) = \frac{\text{E}\big((\hat k - k)^2\big)}{n^2} = \frac{\text{var}(\hat k)}{n^2} = \frac{\epsilon (1 - \epsilon)}{n}
$$

The same result is obtain with the standard error propagation formula, which is exact in this case, since $\epsilon$ is a linear function of $k$. We performed the calculation from first principles instead of using this formula, because it is instructive for the next step, where $\epsilon$ becomes a non-linear function.

## No background, n Poisson distributed

To simplify the calculation, we use that the efficiency can be written in terms of two independently Poisson-distributed numbers, the number of successes $n_1$ and failures $n_2$,

$$
\epsilon = \frac{n_1}{n_1 + n_2} = \frac{n_1}{n},
$$

with $n = n_1 + n_2$.

In a particular sample, the sample values $\hat n_k$ will deviate from the expectation values $n_k$ by random offsets $d_k = \hat n_k - n_k$. This has an effect on the estimator $\hat \epsilon(\hat n_1, \hat n_2) = \hat n_1 / (\hat n_1 + \hat n_2)$, that we can evaluate by expanding the formula around the true values in a two-dimensional Taylor series with $\hat n_k = n_k$, treating the integer values $\hat n_k$ like real numbers,

$$
\hat \epsilon = \sum_{i=0}^\infty\sum_{j=0}^\infty \frac {d_1^i \,d_2^j}{i! \, j!} \frac{\partial^{i+j} \hat \epsilon}{\partial \hat n_1^i \partial \hat n_2^j} (n_1, n_2)
$$

```python
def bivariate_series(expr, x, y, dx, dy, order_max):
    return sum(
        sum(
            dx ** i * dy ** j /
            (factorial(i) * factorial(j)) *
            diff(expr, x, i, y, j)
            for i in range(order_max + 1)
            if (i + j) <= order_max
        ) for j in range(order_max + 1)
    )
```

```python
n1, n2, n, epsilon = symbols("n_1 n_2 n epsilon", real=True, nonnegative=True)
d1, d2 = symbols("d_1 d_2", real=True)

eps = n1 / (n1 + n2)

delta_eps = (bivariate_series(eps, n1, n2, d1, d2, 7) - eps)
delta_eps = delta_eps.subs([(n1, epsilon * n), (n2, (1-epsilon) * n)])
delta_eps = delta_eps.as_poly([d1, d2])
delta_eps
```

$\displaystyle \operatorname{Poly}{\left( \frac{1 - \epsilon}{n^{7}} d_{1}^{7} + \frac{6 - 7 \epsilon}{n^{7}} d_{1}^{6}d_{2} + \frac{\epsilon - 1}{n^{6}} d_{1}^{6} + \frac{15 - 21 \epsilon}{n^{7}} d_{1}^{5}d_{2}^{2} + \frac{6 \epsilon - 5}{n^{6}} d_{1}^{5}d_{2} + \frac{1 - \epsilon}{n^{5}} d_{1}^{5} + \frac{20 - 35 \epsilon}{n^{7}} d_{1}^{4}d_{2}^{3} + \frac{15 \epsilon - 10}{n^{6}} d_{1}^{4}d_{2}^{2} + \frac{4 - 5 \epsilon}{n^{5}} d_{1}^{4}d_{2} + \frac{\epsilon - 1}{n^{4}} d_{1}^{4} + \frac{15 - 35 \epsilon}{n^{7}} d_{1}^{3}d_{2}^{4} + \frac{20 \epsilon - 10}{n^{6}} d_{1}^{3}d_{2}^{3} + \frac{6 - 10 \epsilon}{n^{5}} d_{1}^{3}d_{2}^{2} + \frac{4 \epsilon - 3}{n^{4}} d_{1}^{3}d_{2} + \frac{1 - \epsilon}{n^{3}} d_{1}^{3} + \frac{6 - 21 \epsilon}{n^{7}} d_{1}^{2}d_{2}^{5} + \frac{15 \epsilon - 5}{n^{6}} d_{1}^{2}d_{2}^{4} + \frac{4 - 10 \epsilon}{n^{5}} d_{1}^{2}d_{2}^{3} + \frac{6 \epsilon - 3}{n^{4}} d_{1}^{2}d_{2}^{2} + \frac{2 - 3 \epsilon}{n^{3}} d_{1}^{2}d_{2} + \frac{\epsilon - 1}{n^{2}} d_{1}^{2} + \frac{1 - 7 \epsilon}{n^{7}} d_{1}d_{2}^{6} + \frac{6 \epsilon - 1}{n^{6}} d_{1}d_{2}^{5} + \frac{1 - 5 \epsilon}{n^{5}} d_{1}d_{2}^{4} + \frac{4 \epsilon - 1}{n^{4}} d_{1}d_{2}^{3} + \frac{1 - 3 \epsilon}{n^{3}} d_{1}d_{2}^{2} + \frac{2 \epsilon - 1}{n^{2}} d_{1}d_{2} + \frac{1 - \epsilon}{n} d_{1} -  \frac{\epsilon}{n^{7}} d_{2}^{7} + \frac{\epsilon}{n^{6}} d_{2}^{6} -  \frac{\epsilon}{n^{5}} d_{2}^{5} + \frac{\epsilon}{n^{4}} d_{2}^{4} -  \frac{\epsilon}{n^{3}} d_{2}^{3} + \frac{\epsilon}{n^{2}} d_{2}^{2} -  \frac{\epsilon}{n} d_{2}, d_{1}, d_{2}, domain=\mathbb{Z}\left(n, \epsilon\right) \right)}$

To compute the variance $\text{var}(\hat \epsilon)$ we need to square this and compute the expection on both sides. We use that $\text{E}(d_1^i d_2^j) = \text{E}(d_1^i) \, \text{E}(d_2^j)$, since $\hat n_1$ and $\hat n_2$ are independently sampled. The expectation value $\text{E}(d_k^j)$ is the $j$-th central moment of the Poisson distribution with $\lambda = n_k$. These moments are Touchard polynomials, which are conveniently computed with a recursion formula.

```python
# compute mapping d_k ** i -> E[d_k ** i]

def central_moments(n):
    x = symbols("x")
    r = [1]
    for ni in range(n):
        # recursion for Touchard polynomial
        m = x * sum(binomial(ni, k) * r[k] for k in range(ni))
        r.append(m.expand())

    return [lambdify([x], ri) for ri in r]

replacements = central_moments(14)
```

Now we compute $\text{var}(\hat \epsilon) = \text{E}\big((\hat \epsilon - \epsilon)^2\big)$. Since SymPy does not know how to compute an expectation and that it is a linear operator, we need to resort to advanced expression manipulation in the following steps. We use that the expectation is linear and $\text{E}(n_i n_j) = \text{E}(n_i) \, \text{E}(n_j)$ for independent $n_i$ and $n_j$.

```python
def expectation(expr, args, args2, replacements):
    poly = expr.as_poly(args)
    expr2 = []
    for powers, coef in poly.terms():
        x = prod(
            1 if pi == 0 else replacements[pi](argi)
            for pi, argi in zip(powers, args2)
        )
        expr2.append(coef * x)
    return sum(expr2)
```

```python
# replace powers of d_k with the corresponding central moments
expr2 = expectation(delta_eps ** 2, (d1, d2), (epsilon * n, (1-epsilon)*n), replacements)
expr2 = expr2.as_poly(1/n)
expr2
```

$\displaystyle \operatorname{Poly}{\left( \left(- \epsilon^{2} + \epsilon\right) \frac{1}{n^{13}} + \left(- 4081 \epsilon^{2} + 4081 \epsilon\right) \frac{1}{n^{12}} + \left(- 233058 \epsilon^{2} + 233058 \epsilon\right) \frac{1}{n^{11}} + \left(- 1413895 \epsilon^{2} + 1413895 \epsilon\right) \frac{1}{n^{10}} + \left(- 1356075 \epsilon^{2} + 1356075 \epsilon\right) \frac{1}{n^{9}} + \left(- 130095 \epsilon^{2} + 130095 \epsilon\right) \frac{1}{n^{8}} + \left(720 \epsilon^{2} - 720 \epsilon\right) \frac{1}{n^{7}} + \left(118 \epsilon^{2} - 118 \epsilon\right) \frac{1}{n^{6}} + \left(- 24 \epsilon^{2} + 24 \epsilon\right) \frac{1}{n^{5}} + \left(- 6 \epsilon^{2} + 6 \epsilon\right) \frac{1}{n^{4}} + \left(- 2 \epsilon^{2} + 2 \epsilon\right) \frac{1}{n^{3}} + \left(- \epsilon^{2} + \epsilon\right) \frac{1}{n^{2}} + \left(- \epsilon^{2} + \epsilon\right) \frac{1}{n}, \frac{1}{n}, domain=\mathbb{Z}\left[\epsilon\right] \right)}$

We keep only terms up to $O(1/n^2)$.

```python
expr3 = []
for (p, ), coef in expr2.terms():
    if p <= 2:
        expr3.append(coef * n ** -p)
eps_var = sum(expr3).subs(epsilon, n1/n).subs(n, n1 + n2).simplify()
eps_var
```

$\displaystyle \frac{n_{1} n_{2} \left(n_{1} + n_{2} + 1\right)}{\left(n_{1} + n_{2}\right)^{4}}$

Up to the extra factor $(n + 1)/n$, this is the variance of the binomial distribution we derived previously.

This formula is accurate for $n > 10$. To get accurate results at even smaller $n$, we need to include to higher orders.

```python
expr3 = []
for (p, ), coef in expr2.terms():
    if p <= 4:
        expr3.append(coef * n ** -p)
eps_var = sum(expr3).subs(epsilon, n1/n).subs(n, n1 + n2).simplify()
eps_var
```

$\displaystyle \frac{n_{1} n_{2} \left(2 n_{1} + 2 n_{2} + \left(n_{1} + n_{2}\right)^{3} + \left(n_{1} + n_{2}\right)^{2} + 6\right)}{\left(n_{1} + n_{2}\right)^{6}}$

This formula is accurate down to $n \ge 5$.

The values $n_k$ that appear in the formula are the unknown expectation values and need to be replaced by their sample estimates $\hat n_k$ in practice. This is an application of the plug-in principle on which also bootstrap theory is based. There is a small bias involved in doing this, but it is negligible here.

```python
eps_var_py = lambdify((n1, n2), eps_var)
```

```python
rng = np.random.default_rng(1)
eps = np.linspace(0, 1, 100)
data = {}
for n in (5, 10, 20, 50, 100, 1000):
    s1 = []
    s2 = []
    for epsi in eps:
        n1 = n * epsi
        n2 = n * (1 - epsi)
        k1 = rng.poisson(n1, size=10000)
        k2 = rng.poisson(n2, size=10000)
        ma = k1 + k2 > 0
        k1 = k1[ma]
        k2 = k2[ma]
        s1.append(np.std(k1 / (k1 + k2)))
        # it is important to average the variance instead of the standard deviation
        s2.append(np.mean(eps_var_py(k1, k2)) ** 0.5)
    data[n] = np.array((eps, s1, s2))
```

```python
plt.figure(figsize=(10, 4))
for n, (eps, s1, s2) in data.items():
    l, = plt.plot(eps, s1, ":", label=f"MC, {n=}")
    plt.plot(eps, s2, "--", color=l.get_color(), label=f"formula, {n=}")
    print(f"{n=} median deviation {np.median(s1[s2 > 0] / s2[s2 > 0]) - 1:.2f}")
plt.subplots_adjust(right=0.6)
plt.xlabel(r"$\epsilon$")
plt.ylabel(r"$\sigma(\epsilon)$")
plt.figlegend(loc=(0.69, 0.2));
```

### Wilson interval

To-do.

## With background

We now assume that the estimates $\hat n_1$ and $\hat n_2$ cannot obtained by direct counting, but need to be estimated from a fit that inclues a signal and background component. If the fit is unbiased, the average contribution from the background is zero, which we will assume, but it enlargens the uncertainty of the estimate.

We can model this by adding a random contribution from the background to our estimate,

$$
\hat n = n + z_1 \sigma_\text{poisson} + z_2 \sigma_\text{bkg} = n + z_1 \sqrt{n} + z_2 \sigma_\text{bkg},
$$

where $z_1$ and $z_2$ are independent random variables with zero mean and unit variance.

The expected variance of $\hat n$ then is

$$
\sigma^2(\hat n) = n + \sigma^2_\text{bkg}.
$$

In practice, estimating $\sigma^2_\text{bkg}$ directly is difficult, but fortunately, the left-hand side $\sigma^2(\hat n)$ can be estimated from the fit (MINUITs Hesse error). Thus we can compute an estimate for the unknown $\sigma^2_\text{bkg}$ with the formula,

$$
\sigma^2_\text{bkg} = \sigma^2(\hat n) -  n.
$$

In practice, we have to replace the expectation values $\sigma^2(\hat n)$ and $n$ with the corresponding estimates from the fit.

We insert this into the formula for $\hat \epsilon$ and get:

$$
\hat \epsilon = \frac{\hat n_1}{\hat n_1 + \hat n_2} = \frac{n_1 + z_1 \sqrt{n_1} + z_2 \sigma_\text{bkg,1}}{n_1 + n_2 + z_1 \sqrt{n_1} + z_2 \sigma_\text{bkg,1} + z_3 \sqrt{n_2} + z_4 \sigma_\text{bkg,2}}
$$

As before, the variance of $\hat \epsilon$ is computed via standard error propagation, using that the $z_i$ are independent and standard normal. This reflects the assumption that the background in both fits is independently sampled, which in practice is usually not exactly true. If the correlation can be estimated, it can be trivially included in the calculation, but I will not do this here.

$$
\sigma^2(\hat \epsilon) = \left[\left(\frac{\text{d}\hat \epsilon}{\text{d}z_1}\right)^2 + \left(\frac{\text{d}\hat \epsilon}{\text{d}z_2}\right)^2 +
\left(\frac{\text{d}\hat \epsilon}{\text{d}z_3}\right)^2 +
\left(\frac{\text{d}\hat \epsilon}{\text{d}z_4}\right)^2
\right]_{z_1 = 0,\, z_2=0,\,z_3=0,\,z_4=0}.
$$

```python
(n1, n2, sigma1, sigma2,
 sigma_bkg1, sigma_bkg2,
 z1, z2, z3, z4) = symbols(
    "n_1 n_2 sigma_1 sigma_2 "
    "sigma_bkg_1 sigma_bkg_2 "
    "z_1 z_2 z_3 z_4", real=True)

g = (n1 + z1 * sqrt(n1) + z2 * sigma_bkg1) / \
    (n1 + n2
     + z1 * sqrt(n1) + z2 * sigma_bkg1
     + z3 * sqrt(n2) + z4 * sigma_bkg2); g
```

$\displaystyle \frac{\sqrt{n_{1}} z_{1} + n_{1} + \sigma_{bkg 1} z_{2}}{\sqrt{n_{1}} z_{1} + n_{1} + \sqrt{n_{2}} z_{3} + n_{2} + \sigma_{bkg 1} z_{2} + \sigma_{bkg 2} z_{4}}$

```python
g = g.subs([(sigma_bkg1, sqrt(sigma1 ** 2 - n1)),
            (sigma_bkg2, sqrt(sigma2 ** 2 - n2))]); g
```

$\displaystyle \frac{\sqrt{n_{1}} z_{1} + n_{1} + z_{2} \sqrt{- n_{1} + \sigma_{1}^{2}}}{\sqrt{n_{1}} z_{1} + n_{1} + \sqrt{n_{2}} z_{3} + n_{2} + z_{2} \sqrt{- n_{1} + \sigma_{1}^{2}} + z_{4} \sqrt{- n_{2} + \sigma_{2}^{2}}}$

```python
g_var = sum(diff(g, z) ** 2 for z in (z1, z2, z3, z4))
g_var = g_var.subs([(z1, 0), (z2, 0), (z3, 0), (z4, 0)])
g_var = g_var.simplify(); g_var
```

$\displaystyle \frac{n_{1}^{2} \sigma_{2}^{2} + n_{2}^{2} \sigma_{1}^{2}}{\left(n_{1} + n_{2}\right)^{4}}$

We thus obtain

$$
\sigma(\hat \epsilon) = \frac{\sqrt{
n_1^2 \, \sigma^2(\hat n_2) +
n_2^2 \, \sigma^2(\hat n_1)
}}{n^2},
$$

with $n = n_1 + n_2$. In practice, $n_1$, $\sigma^2(\hat n_1)$, ...  need to be replaced with their fit estimates.

As a check, I show how one can recover the previous formula, which is a special case, by setting $\sigma^2(\hat n_k) = n_k$ (which means no background contribution),

$$
\sigma^2(\hat \epsilon) = \frac{
n_1^2 \, \sigma^2(\hat n_2) +
n_2^2 \, \sigma^2(\hat n_1)
}{n^4} =
\frac{
n_1^2 \, n_2 +
n_2^2 \, n_1
}{n^4} =
\frac{
n_1 \, n_2 \, (n_1 + n_2)
}{n^4} =
\frac{
n_1 \, n_2
}{n^3}.
$$

We again check the formula against Monte-Carlo simulations.

```python
g_var_py = lambdify([n1, sigma1, n2, sigma2], g_var)
```

```python
def model(x, b0, b1, b2, s, mu, sigma):
    nb = bernstein.scaled_cdf(x, (b0, b1, b2), -1, 1)
    ns = s * norm.cdf(x, mu, sigma)
    return ns + nb

@joblib.delayed
def run(eps, n):
    rng = np.random.default_rng(1)

    t1 = []
    t2 = []
    for itry in range(100):
        sigma = 0.1
        results = []
        for p in (eps, 1 - eps):
            s = rng.normal(0, sigma, size=rng.poisson(p * n))
            b = rng.uniform(-1, 1, size=n // 5)
            x = np.append(s, b)
            w, xe = np.histogram(x, bins=50, range=(-1, 1))
            cost = ExtendedBinnedNLL(w, xe, model)
            cost += NormalConstraint(("mu", "sigma"), (0, sigma), (1, 1))
            s_start = eps * n
            b_start = n
            m = Minuit(cost,
                       b0=b_start, b1=b_start, b2=b_start,
                       s=s_start + 1, mu=0, sigma=sigma)
            m.limits[:4] = (0, None)
            m.limits["mu"] = (-0.5, 0.5)
            m.limits["sigma"] = (0.02, 1)
            m.migrad()
            if m.valid and m.accurate:
                results.append((m.values["s"], m.errors["s"]))
            else:
                print(n, f"{eps:.2f}", itry, "signal" if p == eps else "background",
                      "fit failed")
        if len(results) < 2:
            continue

        (n_pass, sigma_pass), (n_fail, sigma_fail) = results

        t1i = n_pass / (n_pass + n_fail)
        t2i = g_var_py(n_pass, sigma_pass, n_fail, sigma_fail)
        t1.append(t1i)
        t2.append(t2i)

    s1 = np.std(t1)
    s2 = np.mean(t2) ** 0.5
    return s1, s2

data = {}
eps = np.linspace(0, 1, 20)
with joblib.Parallel(-1) as pool:
    for n in (50, 100, 1000):
        s1, s2 = np.transpose(pool(run(epsi, n) for epsi in eps))
        data[n] = (eps, s1, s2)
```

```python
plt.figure(figsize=(10, 4))
for n, (eps, s1, s2) in data.items():
    l, = plt.plot(eps, s1, ":", label=f"MC, {n=}")
    plt.plot(eps, s2, "--", color=l.get_color(), label=f"formula, {n=}")
    print(f"{n=} median deviation {np.median(s1[s2 > 0] / s2[s2 > 0]) - 1:.2f}")
plt.subplots_adjust(right=0.6)
plt.xlabel(r"$\epsilon$")
plt.ylabel(r"$\sigma(\epsilon)$")
plt.figlegend(loc=(0.69, 0.2));
```

Again, the derived formula does reasonably well with deviations around 5 % for $n > 50$. This is usually sufficient to calculate error bars for efficiency plots. There should be some bias for small $n$ as before, but it is difficult to see here due to other larger random scatter, since I generated less toys to keep the computation time on my laptop reasonable.

Finally, I compare the uncertainty estimates for various levels of background. The estimate with zero background corresponds to the estimate which one obtains with the naive formula.

```python
plt.figure(figsize=(14, 4))
eps = np.linspace(0, 1, 100)
for n in (10, 50, 100):
    n_pass = n * eps
    n_fail = n * (1 - eps)
    n_bkg = 0
    col = None
    for i, n_bkg in enumerate((0, 1, 5, 10)):
        s2 = g_var_py(n_pass, (n_pass + n_bkg) ** 0.5, n_fail, (n_fail + n_bkg) ** 0.5) ** 0.5
        col = plt.plot(eps, s2, "--" if i > 0 else ":", color=col,
                       lw=0.5 * i + 1.5, label=f"{n=} {n_bkg=}")[0].get_color()
plt.subplots_adjust(right=0.5)
plt.figlegend(loc=(0.64, 0.2))
plt.xlabel(r"$\epsilon$")
plt.ylabel(r"$\sigma(\hat \epsilon)$")
plt.semilogy();
```

The error of using a formula that neglects the background contribution is largest when the efficiency is close to zero or one.

These are frequently the interesting cases. For example, the [tracking efficency of the LHCb experiment]((https://doi.org/10.1088/1748-0221/10/02/P02007)) is estimated from fits to decays in calibration samples and the efficiency is around 96 %, so that the difference in the formulas may impact the estimated uncertainty.

#### Formula for fits to selected and total events

Sometimes it is not easy to split the two samples into failed and passed cases, one has instead fits to the passed and total cases.

In this case, we need to replace $n_2$ and $\sigma^2(\hat n_2)$ in the main formula for $\sigma(\hat \epsilon)$, since we then only have $n = n_1 + n_2$, $\sigma^2(\hat n)$, $n_1$, and $\sigma^2(\hat n_1)$ available.

Since both estimates $\hat n_1$ and $\hat n_2$ are completely independent (again assuming independent background), their sum $\hat n = \hat n_1 + \hat n_1$ must have the variance $\sigma^2(\hat n) = \sigma^2(\hat n_1) + \sigma^2(\hat n_2)$.

Thus the formula for this case becomes

$$
\sigma(\hat \epsilon) = \frac{\sqrt{
n_1^2 \, (\sigma^2(\hat n) - \sigma^2(\hat n_1)) +
(n - n_1)^2 \, \sigma^2(\hat n_1)
}}{n^2}.
$$

Uncertainty of efficiency computed from fitted yields.ipynb


# Sample Code

Run this script. Edit it and make it your own.

```

# sympy_examples.py
# -------------------------------------------------------------------------
# Demonstrate some useful methods available in the SymPy module.
# -------------------------------------------------------------------------
from sympy import *
init_session()

expand( (x + y)**5 )
factor( x**6 - 1 )
pi.n(100)
plot(besselj(0, x), besselj(1, x), (x, 0, 10))
diff( x*sin(y), x, y )
integrate(cos(x)**2, x)
integrate(exp(-x**2), (x, -oo, oo))
Sum(k**3, (k, 0, m)).doit().factor()
dsolve(f(x).diff(x) + f(x) - x, f(x)).simplify()

```


# Assignment Problem

A fundamental combinatorial optimization problem.

- $n$ tasks to be done
- $n$ workers to do the tasks
- Each worker has a certain cost for each task ($c_{ij}$)

Problem: Find the best assignment of all tasks that minimizes the total cost.

Constraints:
* Each worker can do at most one task
* All tasks need to be done

# Mathematical Formulation

\begin{align}
\text{minimize} & \sum_{i = 1}^{n}\sum_{j = 1}^{n} c_{ij}x_{ij}, \\
\text{subject to} & \\
& \sum_{i = 1}^{n} x_{ij} \leq 1 & \forall j = 1, ..., n & \ \ \textit{ (workload)},\\
& \sum_{j = 1}^{n} x_{ij} = 1 & \forall i = 1, ..., n & \ \ \textit{ (task completion)}, \\
& x_{ij} \in \{0,1\} & \forall i,j = 1, ..., n
\end{align}

where $x_{ij}$ is 1 if worker $i$ performs task $j$, 0 otherwise.

We can replace the integrality constraints with continuous constraints. The solution will not change as the constraint matrix is totally unimodular.

# Coding in Python using gurobipy
## Step 1: Creating input data (cost matrix)

```python
import numpy as np
cost = np.random.randint(1, 10, (4,4))
```

## Step 2: Importing gurobipy package

```python
from gurobipy import Model, GRB
```

## Step 3: Creating the model

```python
assignment_model = Model('Assignment')
```

## Step 4: Creating decision variables

```python
x = assignment_model.addVars(cost.shape[0],
                            cost.shape[1],
                            vtype = GRB.BINARY,
                            name = "x")
```

## Step 5: Adding the constraints
\begin{align}
\sum_{i = 1}^{n} x_{ij} \leq 1 &\qquad \forall j = 1, ..., n & \ \ \textit{ (workload)},\\
\sum_{j = 1}^{n} x_{ij} = 1 & \qquad \forall i = 1, ..., n & \ \ \textit{ (task completion)}.
\end{align}

```python
# sum_{i = 1}^{n} x_{ij} <= 1 for all j
assignment_model.addConstrs((sum(x[i,j] for i in range(cost.shape[0])) <= 1
                             for j in range(cost.shape[1])),
                            name = 'work_load')

# sum_{j = 1}^{n} x_{ij}  = 1 for all i
assignment_model.addConstrs((sum(x[i,j] for j in range(cost.shape[1])) == 1
                             for i in range(cost.shape[0])),
                            name = 'task_completion')
```

    {0: <gurobi.Constr *Awaiting Model Update*>,
     1: <gurobi.Constr *Awaiting Model Update*>,
     2: <gurobi.Constr *Awaiting Model Update*>,
     3: <gurobi.Constr *Awaiting Model Update*>}

## Step 6: Defining the objective function
$$\sum_{i = 1}^{n}\sum_{j = 1}^{n} c_{ij}x_{ij}$$

```python
obj_fn = sum(cost[i,j]*x[i,j] for i in range(cost.shape[0]) for j in range(cost.shape[1]))

assignment_model.setObjective(obj_fn, GRB.MINIMIZE)
```

## Step 7: Solve and inspect the model

```python
assignment_model.setParam('OutputFlag', False)
assignment_model.optimize()

print('Model Statistics')
assignment_model.printStats()
print('\n\nModel Output\n')
print(assignment_model.display())
```

    Model Statistics

    Statistics for model Assignment :
      Linear constraint matrix    : 8 Constrs, 16 Vars, 32 NZs
      Variable types              : 0 Continuous, 16 Integer (16 Binary)
      Matrix coefficient range    : [ 1, 1 ]
      Objective coefficient range : [ 1, 9 ]
      Variable bound range        : [ 1, 1 ]
      RHS coefficient range       : [ 1, 1 ]

    Model Output

    Minimize
       <gurobi.LinExpr: 2.0 x[0,0] + 3.0 x[0,1] + 6.0 x[0,2] + 3.0 x[0,3] + 3.0 x[1,0] + 9.0 x[1,1] + 9.0 x[1,2] + x[1,3] + x[2,0] + x[2,1] + 8.0 x[2,2] + 3.0 x[2,3] + 7.0 x[3,0] + 2.0 x[3,1] + 9.0 x[3,2] + 4.0 x[3,3]>
    Subject To
       work_load[0] : <gurobi.LinExpr: x[0,0] + x[1,0] + x[2,0] + x[3,0]> <= 1.0
       work_load[1] : <gurobi.LinExpr: x[0,1] + x[1,1] + x[2,1] + x[3,1]> <= 1.0
       work_load[2] : <gurobi.LinExpr: x[0,2] + x[1,2] + x[2,2] + x[3,2]> <= 1.0
       work_load[3] : <gurobi.LinExpr: x[0,3] + x[1,3] + x[2,3] + x[3,3]> <= 1.0
       task_completion[0] : <gurobi.LinExpr: x[0,0] + x[0,1] + x[0,2] + x[0,3]> = 1.0
       task_completion[1] : <gurobi.LinExpr: x[1,0] + x[1,1] + x[1,2] + x[1,3]> = 1.0
       task_completion[2] : <gurobi.LinExpr: x[2,0] + x[2,1] + x[2,2] + x[2,3]> = 1.0
       task_completion[3] : <gurobi.LinExpr: x[3,0] + x[3,1] + x[3,2] + x[3,3]> = 1.0
    Binaries
       ['x[0,0]', 'x[0,1]', 'x[0,2]', 'x[0,3]', 'x[1,0]', 'x[1,1]', 'x[1,2]', 'x[1,3]', 'x[2,0]', 'x[2,1]', 'x[2,2]', 'x[2,3]', 'x[3,0]', 'x[3,1]', 'x[3,2]', 'x[3,3]']
    None

## Output the solution

```python
print('Optimization is done. Objective Function Value: %.2f' % assignment_model.objVal)

# Get values of the decision variables
for v in assignment_model.getVars():
    if v.x > 0:
        print('%s: %g' % (v.varName, v.x))
```

    Optimization is done. Objective Function Value: 10.00
    x[0,2]: 1
    x[1,3]: 1
    x[2,0]: 1
    x[3,1]: 1

# Relaxing the binary constraint

```python
assignment_model_2 = Model('Assignment_2')

y = assignment_model_2.addVars(cost.shape[0],
                               cost.shape[1],
                               vtype = GRB.CONTINUOUS,
                               lb = 0, ub = 1,
                               name = "x")

# sum_{i = 1}^{n} y_{ij} <= 1 for all j
assignment_model_2.addConstrs((sum(y[i,j] for i in range(cost.shape[0])) == 1
                               for j in range(cost.shape[1])),
                              name = 'work_load')

# sum_{j = 1}^{n} y_{ij}  = 1 for all i
assignment_model_2.addConstrs((sum(y[i,j] for j in range(cost.shape[1])) == 1
                               for i in range(cost.shape[0])),
                              name = 'task_completion')

obj_fn = sum(cost[i,j]*y[i,j] for i in range(cost.shape[0]) for j in range(cost.shape[1]))
assignment_model_2.setObjective(obj_fn, GRB.MINIMIZE)

assignment_model_2.setParam('OutputFlag', False)

assignment_model_2.optimize()

print('Optimization is done. Objective Function Value: %.2f' % assignment_model_2.objVal)

# Get values of the decision variables
for v in assignment_model.getVars():
    if v.x > 0:
        print('%s: %g' % (v.varName, v.x))
```

    Optimization is done. Objective Function Value: 10.00
    x[0,2]: 1
    x[1,3]: 1
    x[2,0]: 1
    x[3,1]: 1

```python

```