### Tables of Content

#### Linear Algebra Tools

1. Operator Matrices
    - Pauli: I, X, Y, Z
    - Hadamard: H
    - Phase: P
    - Sqrt(X): SX
    - Sqrt(Z): S
    - Sqrt(H): SH
    - 4rt (Z): T
    - X root: Xrt(s)
    - H root: Hrt(s)
    - Rotation Matrices: Rx($\theta$), Ry($\theta$), Rz($\theta$)
    - U3 Matrix: U3($\theta, \phi, \lambda$)
    - Controlled-Not: CX

</br>

2. Common Statevectors
    - $|0\rangle$: zero
    - $|1\rangle$: one
    - $|+\rangle$: plus
    - $|-\rangle$: minus
    - $| \uparrow \rangle$: up
    - $| \downarrow \rangle$: down
    - Bell States: B00, B01, B10, B11

</br>

3. Lambda Methods
    - ndarray to list: to_list(array)
    - tensor: *****initial_state
    - matmul: *****initial_state

</br>

4. Full Methods
    - Calculate Hermitian Conjugate: dagger(mat)
    - Build CU matrix: cu_matrix(no_qubits, control, target, U, little_edian)
    - Find RX, RY for arbitrary U3: angles_from_state_vectors(output_statevector)

</br>

5. Visualizations
    - view(mat, rounding = 10)

#### Qiskit Tools

1. Linear Algebra
    - Short-hand QC: q(*****regs, name=None, global_phase=0)
    - Multi-controlled Unitary: control_unitary(circ, unitary, *****controls, target)
    - Control Phase: control_phase(circ, angle, control_bit, target_bit, recip=True, pi_on=True)

</br>

2. Visualizations
    - Draw Circuit: milk(circ)
    - Draw Transpiled Circuit: dtp(circ, print_details = True, visual = True, return_values = False)
    - Get Unitary / Statevector Function: get(circ, types = 'unitary', nice = True)
    - Displaying Histogram / Bloch / Counts: sim(circ, visual = 'hist')

</br>

3. Toffoli Optimizaton Specific
    - Unitary Checker: unitary_check(test_unitary)
    - Multi-Hadamard Composition: h_relief(n, no_h)

### Import

```python
import numpy as np
import sympy as sp
from sympy.solvers.solveset import linsolve

import matplotlib
import matplotlib.pyplot as plt
matplotlib.use('Agg')

from sympy import Matrix, init_printing

import qiskit
from qiskit import *
from qiskit.aqua.circuits import *

# Representing Data
from qiskit.providers.aer import QasmSimulator, StatevectorSimulator, UnitarySimulator
from qiskit.tools.visualization import plot_histogram, plot_state_city, plot_bloch_multivector

# Monitor Job on Real Machine
from qiskit.tools.monitor import job_monitor

from functools import reduce # perform sucessive tensor product

# Calculating cost
from sklearn.metrics import mean_squared_error

# Generating random unitary matrix
from scipy.stats import unitary_group

# Measure run time
import time

# Almost Equal
from numpy.testing import assert_almost_equal as aae
```

    Duplicate key in file '/Users/minhpham/.matplotlib/matplotlibrc' line #2.
    Duplicate key in file '/Users/minhpham/.matplotlib/matplotlibrc' line #3.

### Linear Algebra Tools

```python
# Matrices
I = np.array([[1, 0], [0, 1]])
X = np.array([[0, 1], [1, 0]])
Y = np.array([[0, -1j], [1j, 0]])
Z = np.array([[1, 0], [0, -1]])
H = 1/np.sqrt(2)*np.array([[1, 1], [1, -1]])
P = lambda theta: np.array([[1, 0], [0, np.exp(1j*theta)]])

# sqrt(X)
SX = 1/2 * np.array([[1+1j, 1-1j], [1-1j, 1+1j]])

# sqrt(Z)
S = np.array([[1, 0], [0, 1j]])

# sqrt(H)
SH = (1j/4-1/4)*np.array([[np.sqrt(2) + 2j, np.sqrt(2)], [np.sqrt(2), -np.sqrt(2)+2j]])

# 4th root of Z
T = np.array([[1, 0], [0, 1/np.sqrt(2) + 1/np.sqrt(2)*1j]])

# X power
Xp = lambda t: 1/2 * np.array([[1, 1], [1, 1]]) + np.exp(1j*np.pi*t)/(2) * np.array([[1, -1], [-1, 1]])

# H power
Hp = lambda t: np.exp(-1j*np.pi*t/2) * np.array([[np.cos(np.pi*t/2) + 1j/np.sqrt(2)* np.sin(np.pi*t/2), 1j/np.sqrt(2) * np.sin(np.pi*t/2)],
                                                   [1j/np.sqrt(2) * np.sin(np.pi*t/2), np.cos(np.pi*t/2)-1j/np.sqrt(2)* np.sin(np.pi*t/2)]])

CX = np.array([[1, 0, 0, 0], [0, 0, 0, 1], [0, 0, 1, 0], [0, 1, 0, 0]])

# Rn Matrix Function
Rx = lambda theta: np.array([[np.cos(theta/2), -1j*np.sin(theta/2)], [-1j*np.sin(theta/2), np.cos(theta/2)]])
Ry = lambda theta: np.array([[np.cos(theta/2), -np.sin(theta/2)], [np.sin(theta/2), np.cos(theta/2)]])
Rz = lambda theta: np.array([[np.exp(-1j*theta/2), 0], [0, np.exp(1j*theta/2)]])

# U3 Matrix
U3 = lambda theta, phi, lam: np.array([[np.cos(theta/2), -np.exp(1j*lam)*np.sin(theta/2)],
                                       [np.exp(1j*phi)*np.sin(theta/2), np.exp(1j*lam + 1j*phi)*np.cos(theta/2)]])

# Eigenvectors of Pauli Matrices
zero = np.array([[1], [0]]) # Z plus basis state
one = np.array([[0], [1]]) # Z plus basis state

plus = np.array([[1], [1]])/np.sqrt(2) # X plus basis state
minus = np.array([[1], [-1]])/np.sqrt(2) # X minus basis state

up = np.array([[1], [1j]])/np.sqrt(2) # Y plus basis state
down = np.array([[1], [-1j]])/np.sqrt(2) # Y plus basis state

# Bell States
B00 = np.array([[1], [0], [0], [1]])/np.sqrt(2) # Bell of 00
B01 = np.array([[1], [0], [0], [-1]])/np.sqrt(2) # Bell of 01
B10 = np.array([[0], [1], [1], [0]])/np.sqrt(2) # Bell of 10
B11 = np.array([[0], [-1], [1], [0]])/np.sqrt(2) # Bell of 11

# ndarray to list
to_list = lambda array: list(np.squeeze(array))

# Tensor Product of 2+ matrices/ vectors
tensor = lambda *initial_state: reduce(lambda x, y: np.kron(x, y), initial_state)

# Matrix Multiplicaton of 2+ matrices / vectors
mat_mul = lambda *initial_state: reduce(lambda x, y: np.dot(x, y), initial_state)
```

###### Calculate Hermitian Conjugate

```python
def dagger(mat):

    # Calculate Hermitian conjugate
    mat_dagger = np.conj(mat.T)

    # Assert Hermitian identity
    aae(np.dot(mat_dagger, mat), np.identity(mat.shape[0]))

    return mat_dagger
```

###### CU Matrix

```python
def cu_matrix(no_qubits, control, target, U, little_edian = True):

    """
    Manually build the unitary matrix for non-adjacent CX gates

    Parameters:
    -----------
    no_qubits: int
        Number of qubits in the circuit
    control: int
        Index of the control qubit (1st qubit is index 0)
    target: int
        Index of the target qubit (1st qubit is index 0)
    U: ndarray
        Target unitary matrix
    edian: bool (True: qiskit convention)
        Qubits order convention

    Returns:
    --------
    cx_out:
        Unitary matrix for CU gate
    """

    left = [I]*no_qubits
    right = [I]*no_qubits

    left[control] = np.dot(zero, zero.T)
    right[control] = np.dot(one, one.T)

    right[target] = U

    if little_edian:
        cx_out = tensor(*reversed(left)) + tensor(*reversed(right))
    else:
        cx_out = tensor(*left) + tensor(*right)

    # This returns a unitary in qiskit 'little eddian', to switch back, simply switch the target for control

    return cx_out
```

###### Angles from Statevector

```python
def angles_from_statevectors(output_statevector):

    """
    Calculate correct x, y rotation angles from an arbitrary output statevector

    Paramters:
    ----------
    output_statevector: ndarray
        Desired output state

    Returns:
    --------
    phi: float
        Angle to rotate about the y-axis [0, 2pi)
    theta: float
        Angle to rotate about the x-axis [0, 2pi)

    """

    # Extract the components
    x, z = output_statevector.real
    y, w = output_statevector.imag

    # Calculate the correct angles
    phi = 2*np.arctan2(z,x)[0]
    theta = 2*np.arctan2(y,z)[0]

    print(f'phi: {phi}')
    print(f'theta: {theta}')

    return phi, theta
```

###### View Matrix

```python
def view(mat, rounding = 10):
    display(Matrix(np.round(mat, rounding)))
```

### Qiskit Tools

###### Short-hand Qiskit Circuit

```python
q = lambda *regs, name=None, global_phase=0: QuantumCircuit(*regs, name=None, global_phase=0)
```

###### Controlled Unitary

```python
def control_unitary(circ, unitary, controls, target):

    """
    Composed a multi-controlled single unitary target gate

    Parameters:
    -----------
    circ: QuantumCircuit
        Qiskit circuit of appropriate size, no less qubit than the size of the controlled gate
    unitary: ndarray of (2, 2)
        Unitary operator for the target qubit
    controls: list
        Indices of controlled qubit on the original circuit
    target: int
        Index of target bit

    Returns:
    --------
    new_circ: QuantumCircuit
        Composed circuit with unitary target
    """

    # Get info about circuit parameters

    no_controls = len(controls)
    unitary_size = np.log2(len(unitary))

    # Build unitary circuit

    qc = QuantumCircuit(unitary_size)
    qc.unitary(unitary, range(int(unitary_size)))
    qc = qc.control(no_controls)

    # Composed the control part in the circuit

    new_circ = circ.compose(qc, (*controls, target))

    return new_circ
```

##### Controlled Phase

```python
def control_phase(circ, angle, control_bit, target_bit, recip = True, pi_on = True):

    """
    Add a controlled-phase gate

    Parameters:
    -----------
    circ: QuantumCircuit
        Inputted circuit

    angle: float
        Phase Angle

    control_bit: int
        Index of control bit

    target_bit: int
        Index of target bit

    recip: bool (True)
        Take the reciprocal of the angle

    pi_on: bool (True)
        Multiply pi to the angle

    Returns:
    --------
    circ: QuantumCircuit
        Circuit with built-in CP

    """

    if recip:
        angle = 1/angle
    if pi_on:
        angle *=np.pi

    circ.cp(angle, control_bit, target_bit)

    return circ
```

###### Draw Circuit

```python
def milk(circ):
    return circ.draw('mpl')
```

###### Draw Transpiled Circuit

```python
def dtp(circ, print_details = True, nice = True, return_values = False):

    """
    Draw and/or return information about the transpiled circuit

    Parameters:
    -----------
    circ: QuantumCircuit
        QuantumCircuit to br transpiled
    print_details: bool (True)
        Print the number of u3 and cx gates used
    nice: bool (True)
        Show the transpiled circuit
    return_values: bool (True)
        Return the number of u3 and cx gates used

    Returns:
    --------
    no_cx: int
        Number of cx gates used
    no_u3: int
        Number of u3 gates used

    """

    # Transpile Circuit
    circ = transpile(circ, basis_gates= ['u3', 'cx'], optimization_level=3)

    # Count operations
    gates = circ.count_ops()

    # Compute cost
    try:
        no_u3 = gates['u3']
    except:
        no_u3 = 0

    try:
        no_cx = gates['cx']
    except:
        no_cx = 0

    cost = no_u3 + 10*no_cx

    if print_details:
        # Print Circuit Details
        print(f'cx: {no_cx}')

        print(f'u3: {no_u3}')
        print(f'Total cost: {cost}')

    if nice:
        return circ.draw('mpl')

    if return_values:
        return no_cx, no_u3
```

###### Get Unitary/StateVector Function

```python
def get(circ, types = 'unitary', nice = True):

    """
    This function return the statevector or the unitary of the inputted circuit

    Parameters:
    -----------
    circ: QuantumCircuit
        Inputted circuit without measurement gate
    types: str ('unitary')
        Get 'unitary' or 'statevector' option
    nice: bool
        Display the result nicely option or just return unitary/statevector as ndarray

    Returns:
    --------
    out: ndarray
        Outputted unitary of statevector

    """

    if types == 'statevector':
        backend = BasicAer.get_backend('statevector_simulator')
        out = execute(circ, backend).result().get_statevector()
    else:
        backend = BasicAer.get_backend('unitary_simulator')
        out = execute(circ, backend).result().get_unitary()

    if nice:
        display(Matrix(np.round(out, 10)))
    else:
        return out
```

###### Displaying Histogram / Bloch / Counts

```python
def sim(circ, visual = 'hist'):

    """
    Displaying output of quantum circuit

    Parameters:
    -----------
    circ: QuantumCircuit
        QuantumCircuit with or without measurement gates
    visual: str ('hist')
        'hist' (counts on histogram) or 'bloch' (statevectors on Bloch sphere) or None (get counts only)

    Returns:
    --------
    counts: dict
        Counts of each CBS state
    """

    # Simulate circuit and display counts on a histogram
    if visual == 'hist':
        simulator = Aer.get_backend('qasm_simulator')
        results = execute(circ, simulator).result()
        counts = results.get_counts(circ)
        plot_histogram(counts)

        return counts

    # Get the statevector and display on a Bloch sphere
    elif visual == 'bloch':
        backend = BasicAer.get_backend('statevector_simulator')
        statevector = execute(circ, backend).result().get_statevector()
        get(circ)
        plot_bloch_multivector(statevector)

    # Just get counts
    else:
        simulator = Aer.get_backend('qasm_simulator')
        results = execute(circ, simulator).result()
        counts = results.get_counts(circ)

        return counts
```

###### Unitary Checker

```python
def unitary_check(test_unitary, perfect = False):

    """
    Check if the CnX unitary is correct

    Parameters:
    -----------
    test_unitary: ndarray
        Unitary generated by the circuit
    perfect: ndarray
        Account for phase difference

    """

    # Get length of unitary

    if not perfect:
        test_unitary = np.abs(test_unitary)

    size = test_unitary.shape[0]

    cx_theory = np.identity(size)

    # Change all the difference
    cx_theory[int(size/2) - 1, size - 1] = 1
    cx_theory[size - 1, int(size/2) - 1] = 1
    cx_theory[int(size/2) -1, int(size/2) -1] = 0
    cx_theory[size - 1, size - 1] = 0

    # Assert Similarity
    aae(cx_theory, test_unitary)

    print('Unitary is correct')
```

# Task: Implementing Improved Multiple Controlled Toffoli

### Abstract

Multiple controlled Toffoli gates are crucial in the implementation of modular exponentiation [4], like that used in Shor's algorithm. In today's practical realm of small number of qubits devices, there is a real need for efficient realization of multiple controlled Toffoli gate for 6 to 10 controls.

Shende and Markov proved that the implementation of the $n$-qubit analogue of the $TOFFOLI$ requires at least $2n \ CNOT$ gates [1]. Currently, the best known upper bound is outlined by Maslov stands at $6n-12$ with the used of $\lceil \frac{n-3}{2} \rceil$ ancilla bits [2]. For implementaion without ancillae, we look at the technique outlined in Corollary 7.6 which has $\Theta(n^2)$ complexity [3]. The aboved mention technique however, still has a high implementation cost for relatively low number of controls. This is due to the high coefficient of the $n^2$ term.

Note that in this notebook, $n$ qubits Toffli gates will simply be referred to as $CnX$ gate where $n$ is the number of control bits.

For this project, we outline a technique for building $CnX$ gate with modulo phase shift whose unitary satisfies $UU = I$. For a few examples from $n = 2$ to $n = 15$, we provided some values to compare and contrast our circuit cost versus that of qiskit. We then postulated with high confidence the complexity of the technique to be $O(2^{\frac{n}{2}})$. Comparing this to the quadratic technique in Corollary 7.6 of [3], we found that our circuits are superior for $n = 7, 8, ..., 11$ . At the end, we offers some possible implementation cases for our technique.

### Motivating the General Circuit

The general $CnX$ gate takes in $n+1$ qubits as inputs ($n$ controls, $1$ target). It's action on a set of qubits $\{q_i\}_{i = 0}^{n}$ is defined as followed.

$$CnX(\{q_i\}_{i = 0}^{n}) = \big{(} \bigwedge_{i = 0}^{n-1} q_i \big{)} \oplus q_n$$

Simply stated, the gate flips the target bit if all the controls are $1$s. For example, for $n = 2$, we have the well-known Toffoli gate

```python
circ = q(3)
circ.ccx(0, 1, 2)
milk(circ)
```

And for higher $n$, $6$ for example, the circuit would take this form.

```python
circ = q(7)
circ.mct(list(range(6)), 6)
milk(circ)
```

The cost for the Qiskit implementation of $CnX$ gate from $n = 2$ to $n = 11$ are listed above in terms of the basic operations ($CX$ and $U3$). Note that the general cost is defined as $10CX + U3$.

n | CX | U3 | General Cost
--- | --- | --- | ---
2 | 6 | 8 | 68
3 | 20 | 22 | 222
4 | 44 | 46 | 486
5 | 92 | 94 | 1014
6 | 188 | 190 | 2070
7 | 380 | 382 | 4182
8 | 764 | 766 | 8406
9 | 1532 | 1534 | 16854
10 | 3068 | 3070 | 33750
11 | 6140 | 6142 | 67542

As outlined in Corolllary 7.1 [3]. The number of $CX$ grows by $3\cdot 2^{n-1} - 4$, and $U3$ grows by $3\cdot 2^{n-1} - 2$. Overall, we see an $O(2^n)$ complexity of the general cost.

Our technique takes advantage of the superposition identity that

$$H Z H = X$$

For an arbitrary $CnX$, we split the control into two groups (one controlled by $H$, and one controlled by $Z$). If we defined the number of control bits on the $H$ gates as $a$, we have the circuit $C(a)H - C(n-a)Z - C(a)H$. An example of $n = 7, a = 3$ is shown below.

```python
circ = q(8)
circ = control_unitary(circ, H, [0, 1, 2], 7)
circ = control_unitary(circ, Z, [3, 4, 5, 6], 7)
circ = control_unitary(circ, H, [0, 1, 2], 7)

milk(circ)
```

The two outer most gates are $C3H$, and the middle gate is $C4Z$. Together they create $C7X$ with a negative phase in 7 columns of the unitary. In general, the number of negative phase in the unitary has the form $2^a - 1$. Although $a$ can be varied, for each $n$, there exists a unique value of $a$ that is optimal for the respective circuit. We run and tested out all the different combination of $n$s and $a$s. And we generate the set of opimal combinations shown below.

n | H-a | CX | U3 | General Cost
--- | --- | --- | --- | ---
2 | 1 | 3 | 4 | 34
3 | 1 | 6 | 7 | 67
4 | 1 | 20 | 25 | 225
5 | 2 | 34 | 53 | 393
6 | 2 | 50 | 72 | 572
7 | 3 | 70 | 101 | 801
8 | 4 | 102 | 143 | 1163
9 | 4 | 146 | 196 | 1656
10 | 4 | 222 | 286 | 2506
11 | 5 | 310 | 395 | 3495

### Implementing the General Circuit

The circuit will be implemented recursively using three base cases. When $n = 1$, when have the $CX$ gate. When $n = 2$, we have the below structure.

```python
milk(CnX(2))
```

$n = 3$

```python
dtp(CnX(3))
```

We sketch the following for the general circuit of $CnX$

We also provide the qiskit code implementation of for the general $CnX$ below. At the end is the list of the best implementation for each CnX gate. To use, simply assign ```best[n] ``` to an object and use like a normal QuantumCircuit. Note that $n$ represents the number of controls in the desired $CnX$.

###### CnX/CnP (Multiple-controlled Not modulo phase shift circuit)

```python
def CnX(n, control_list = None, target = None, circ = None, theta = 1):

    """
    Create a CnX modulo phase shift gate

    Parameters:
    -----------
    n: int
        Number of control bits
    control_list: list
        Index of control bits on inputted circuit (if any)
    target: int
        Index of control bits on inputted circuit (if any)
    circ: QuantumCircuit
        Inputted circuit to compose CnX on
    theta: int
        1/theta power X n-bit controlled circuit

    Returns:
    --------
    circ: QuantumCircuit
        CnX modulo phase shift gate

    """

    # Build New Circuit
    if circ == None:
        circ = q(n+1)
        control_list = list(range(n))
        target = n

    # Base Case
    if n == 1:

        circ.cx(*control_list, target)

        return circ

    if n==2:
        circ.ch(control_list[0], target)
        circ.cz(control_list[1], target)
        circ.ch(control_list[0], target)

        return circ

    if n == 3:
        circ.rcccx(*control_list, target)

        return circ

    # New Case

    # CH
    circ.ch(control_list[0], target)

    # CP2
    circ = control_phase(circ, theta*2, control_list[-1], target)

    # C(n-2)X
    circ = CnX(n-2, control_list[1:-1], control_list[-1], circ)

    # -CP2
    circ = control_phase(circ, -theta*2, control_list[-1], target)

    # C(n-2)X
    circ = CnX(n-2, control_list[1:-1], control_list[-1], circ)

    # CnP
    circ = CnP(n-2, control_list[1:-1], target, circ, theta*2)

    # CH
    circ.ch(control_list[0], target)

    return circ

def CnP(n, control_list = None, target = None, circ = None, theta = 1):

    """
    Create a CnP modulo phase shift gate

    Parameters:
    -----------
    n: int
        Number of control bits
    control_list: list
        Index of control bits on inputted circuit (if any)
    target: int
        Index of control bits on inputted circuit (if any)
    circ: QuantumCircuit
        Inputted circuit to compose CnP on
    theta: int
        1/theta power Z n-bit controlled circuit

    Returns:
    --------
    circ: QuantumCircuit
        CnP modulo phase shift gate

    """

    # Build New Circuit
    if circ == None:
        circ = q(n+1)
        control_list = list(range(n))
        target = n

    # Base Case
    if n ==1:
        circ = control_phase(circ, theta, control_list, target)

        return circ

    # New Case

    # CP
    circ = control_phase(circ, theta*2, control_list[-1], target)

    # C(n-1)X
    circ = CnX(n-1, control_list[:-1], control_list[-1], circ)

    # -CP
    circ = control_phase(circ, -theta*2, control_list[-1], target)

    # C(n-1)X
    circ = CnX(n-1, control_list[:-1], control_list[-1], circ)

    # C(n-1)P
    circ = CnP(n-1, control_list[:-1], target, circ, theta*2)

    return circ
```

###### CnH / Multi-Hadamard Composition

```python
def CnH(n, control_list = None, target = None, circ = None, theta = 1):

    """
    Create a CnH modulo phase shift gate

    Parameters:
    -----------
    n: int
        Number of control bits
    control_list: list
        Index of control bits on inputted circuit (if any)
    target: int
        Index of control bits on inputted circuit (if any)
    circ: QuantumCircuit
        Inputted circuit to compose CnH on
    theta: int
        1/theta power H n-bit controlled circuit

    Returns:
    --------
    circ: QuantumCircuit
        CnH modulo phase shift gate

    """

    # Build New Circuit
    if circ == None:
        circ = q(n+1)
        control_list = list(range(n))
        target = n

    # Base Case

    if n ==1 and theta ==1:
        circ.ch(control_list, target)

        return circ

    if n ==1:
        circ.unitary(cu_matrix(2, 0, 1, Hp(1/theta)), [control_list, target])

        return circ

    # New Case

    # CH
    circ.unitary(cu_matrix(2, 0, 1, Hp(1/(theta*2))), [control_list[-1], target])

    # C(n-1)X
    circ = CnX(n-1, control_list[:-1], control_list[-1], circ)

    # CH
    circ.unitary(cu_matrix(2, 0, 1, Hp(-1/(theta*2))), [control_list[-1], target])

    # C(n-1)X
    circ = CnX(n-1, control_list[:-1], control_list[-1], circ)

    # C(n-1)P
    circ = CnH(n-1, control_list[:-1], target, circ, theta*2)

    return circ

def h_relief(n, no_h, return_circ = False):

    """
    Implementing the general CaH-C(n-a)Z-CaH architecture

    Paramters:
    ----------
    n: int
        Total number of control bits
    no_h: int
        Total number of control bits for the CnH gate
    return_circ: bool
        Return circuit as a QuantumCircuit object

    Returns:
    --------
    circ: QuantumCircuit
        Circuit with CnX and Hadamard Relief

    """

    # n is the number of control qubit
    # no_h is the number of control qubit on the side hadamard
    circ = q(n+1)
    circ= CnH(no_h, list(range(no_h)), n, circ)

    circ = CnP(n-no_h, list(range(no_h, n)), n, circ)
    circ= CnH(no_h, list(range(no_h)), n, circ)

    '''# Test for accuracy
    test = get(circ, nice = False)
    unitary_check(test)'''

    if return_circ:
        return circ

    dtp(circ, nice = False)
```

```python
### List of opimal combinations

best = [None, None, CnX(2), CnX(3), CnX(4), h_relief(5, 2, return_circ = True), h_relief(6, 2, return_circ = True),
 h_relief(7, 3, return_circ = True), h_relief(8, 4, return_circ = True), h_relief(9, 4, return_circ = True),
 h_relief(10, 4, return_circ = True), h_relief(11, 5, return_circ = True), h_relief(12, 6, return_circ = True)]
```

### Postulate for Complexity of the General Cost

We have two lists below showing the number of $U3$ and $CX$ used for the qiskit technique and our technique

```python
## Qiskit

cx_q = np.array([6, 20, 44, 92, 188, 380, 764, 1532, 3068, 6140])
u3_q = np.array([8, 22, 46, 94, 190, 382, 766, 1534, 3070, 6142])

## Our

cx_o = np.array([3, 6, 20, 34, 50, 70, 102, 146, 222, 310])
u3_o = np.array([4, 7, 25, 53, 72, 101, 143, 196, 286, 395])
```

We find the common ratios by taking $a_{n+1}/a_n$, and taking the average of these ratio when $n > 3$ to mitigate the impact of the additive factor.

```python
## Qiskit

rat_1 = cx_q[1:] / cx_q[:-1]
rat_1 = np.mean(rat_1[3:])

rat_2 = u3_q[1:] / u3_q[:-1]
rat_2 = np.mean(rat_2[3:])

## Our

rat_3 = cx_o[1:] / cx_o[:-1]
rat_3 = np.mean(rat_3[3:])

rat_4 = u3_o[1:] / u3_o[:-1]
rat_4 = np.mean(rat_4[3:])
```

```python
rat_1, rat_2, rat_3, rat_4
```

    (2.014071920253367, 2.00693412107478, 1.4460079971764097, 1.398006975531928)

We see that the geometric ratio of our technique is superior to that of qiskit. In base $2$, we can roughly see the following complexity.

$$CX \approx O(1.446^n) \approx O(2^{\frac{n}{2}})$$

$$U3 \approx O(1.380^n) \approx O(2^{\frac{n}{2}})$$

### Compare and Contrast with the $O(n^2)$ technique in Corollary 7.6 of [3]

Lemma 7.5 shows an example of $C8X$ built using 2 $C7X$ and 1 $C7V$. For our purposes, we can assume that the cost of $C7V$ is equal to that of $C7X$. In actuality, the cost of any CnU gate is much greater than that of $CnX$ gates so therefore this assumption gives us a lower bound of the cost of the circuit.

Previous lemmas and corollaries show that these can gates can be broken down further into smaller $C2X$ and $C3X$ gates.

$$\begin{align}C5X &= 12 \ C2X = 12\cdot34 = 408 \\ C7X &= 2 \ C5X + 2 \ C3X = 2\cdot408 + 2\cdot67 = 950 \\ C8X &= 3 \ C7X \end{align}$$

If we let use our implementation of $C2X$ and $C3X$. Then we would have the general cost of $C8X = 2850$. However, as our circuit allow for the use of phase differences, we would also allow this circuit to be used to built bigger examples like shown below.

```python
circ = q(10)
circ = control_unitary(circ, H, [0, 1], 9)
circ.h(9)
circ.mct([2, 3, 4, 5, 6, 7, 8], 9)
circ.h(9)
circ = control_unitary(circ, H, [0, 1], 9)

milk(circ)
```

The $3$ middle gates will have the effect of $C8Z$, and the two gate outside are $C2Z$. This will leads to $C10X$ with phase difference. Now we made one last modification to the implementation of Lemma 7.5. If we look back to the table from before, we can see that our implementation of $C7X$ has a lower than $950$. Because the phase difference does not affect the control operation, we can replace the paper's $C7X$ with ours.

```python
print(1)
dtp(CnH(1), nice = False)
print('\n')
print(2)
dtp(CnH(2), nice = False)
print('\n')
print(3)
dtp(CnH(3), nice = False)
```

    1
    cx: 1
    u3: 2
    Total cost: 12

    2
    cx: 8
    u3: 16
    Total cost: 96

    3
    cx: 18
    u3: 31
    Total cost: 211

Using the $CnH$ implementation cost of $96$. The lower bound of the general cost of $C8X, C9X, C10X$ with and without phase difference are:

n | H-a  | Our Cost | Lower Bound
--- | --- | --- | ---
8 | 0 | 1163 | 2403
9 | 1 | 1656 | 2427
10 | 2 | 2506 | 2595
11 | 3 | 3495 | 2825

This conclusively shows that our technique is superior for $n = 8, 9, 10, 11$. It's easily provable that this supremacy holds for $n = 5, 6, 7$.

### References

1. https://arxiv.org/pdf/0803.2316.pdf
2. https://arxiv.org/pdf/1508.03273.pdf
3. https://arxiv.org/pdf/quant-ph/9503016.pdf
4. https://arxiv.org/abs/quant-ph/9508027

frozenYoghourt/Qbraid - Implementing Improved Multiple Controlled Toffoli.ipynb


```python
from sympy import *
x, C, D = symbols('x C D')
i, j = symbols('i j', integer=True, positive=True)
psi_i = (1-x)**(i+1)
psi_j = psi_i.subs(i, j)
integrand = diff(psi_i, x)*diff(psi_j, x)
integrand = simplify(integrand)
A_ij = integrate(integrand, (x, 0, 1))
A_ij = simplify(A_ij)
print(('A_ij:', A_ij))
f = 2
b_i = integrate(f*psi_i, (x, 0, 1)) - \
      integrate(diff(D*x, x)*diff(psi_i, x), (x, 0, 1)) - \
      C*psi_i.subs(x, 0)
b_i = simplify(b_i)
print(('b_i:', b_i))
N = 1
A = zeros(N+1, N+1)
b = zeros(N+1)
print(('fresh b:', b))
for r in range(N+1):
    for s in range(N+1):
        A[r,s] = A_ij.subs(i, r).subs(j, s)
    b[r,0] = b_i.subs(i, r)
print(('A:', A))
print(('b:', b[:,0]))
c = A.LUsolve(b)
print(('c:', c[:,0]))
u = sum(c[r,0]*psi_i.subs(i, r) for r in range(N+1)) + D*x
print(('u:', simplify(u)))
print(("u'':", simplify(diff(u, x, x))))
print(('BC x=0:', simplify(diff(u, x).subs(x, 0))))
print(('BC x=1:', simplify(u.subs(x, 1))))

```

    ('A_ij:', (i + 1)*(j + 1)/(i + j + 1))
    ('b_i:', ((-C + D)*(i + 2) + 2)/(i + 2))
    ('fresh b:', Matrix([
    [0, 0],
    [0, 0]]))
    ('A:', Matrix([
    [1,   1],
    [1, 4/3]]))
    ('b:', Matrix([
    [  -C + D + 1],
    [-C + D + 2/3]]))
    ('c:', Matrix([
    [-C + D + 2],
    [        -1]]))
    ('u:', C*x - C + D - x**2 + 1)
    ("u'':", -2)
    ('BC x=0:', C)
    ('BC x=1:', D)

```python

```

Data Science and Machine Learning/Machine-Learning-In-Python-THOROUGH/EXAMPLES/FINITE_ELEMENTS/INTRO/SRC/38_U_XX_2_CD.ipynb


# Calc Eigen Value

```python
import numpy as np
```

```python
a=np.array([[16,-1,1,2],
                        [2,12,1,-1],
                        [1,3,-24,2],
                        [4,-2,1,20]],dtype='f')
#initial value s.t. np.linang.norm(x) is 1
x=np.array([0.5,0.5,0.5,0.5]).T
x
```

    array([ 0.5,  0.5,  0.5,  0.5])

```python
epsilon=0.01
maxiteration=100
```

```python
x_new=a @ x
eigen=x_new.dot(x)
x_new/=np.linalg.norm(x_new)
```

```python
for iteration in range(maxiteration):
    x_new =a@x
    eigen=x_new.dot(x)
    if(np.linalg.norm(x_new-eigen*x)<epsilon):
        break
    #normalize
    x_new/=np.linalg.norm(x_new)
    #update x
    x=x_new
else:
    print("fail to calc eigen value")
    exit(1)

print("num of iteration is %d"%iteration)
print("one of eigen value of a is %f"%eigen)
print("its eigen vector is \n{}")

```

    num of iteration is 0
    one of eigen value of a is -24.148327
    its eigen vector is
    [-0.0244012  -0.0268873   0.99911089 -0.02142602]

# final code is...

```python
import numpy as np

a=np.array([[16,-1,1,2],
                        [2,12,1,-1],
                        [1,3,-24,2],
                        [4,-2,1,20]],dtype='f')
#initial value s.t. np.linang.norm(x) is 1
x=np.array([0.5,0.5,0.5,0.5]).T
epsilon=0.01
maxiteration=100
for iteration in range(maxiteration):
    x_new =a@x
    eigen=x_new.dot(x)
    if(np.linalg.norm(x_new-eigen*x)<epsilon):
        break
    #normalize
    x_new/=np.linalg.norm(x_new)
    #update x
    x=x_new
else:
    print("fail to calc eigen value")
    exit(1)

print("num of iteration is %d"%iteration)
print("one of eigen value of a is %f"%eigen)
print("its eigen vector is \n{}".format(x))

```

    num of iteration is 78
    one of eigen value of a is -24.148327
    its eigen vector is
    [-0.0244012  -0.0268873   0.99911089 -0.02142602]

```python
import sympy as sy
sy.init_printing()
sy_a=sy.Matrix(a)
sy_a
```

```python
sy_a.eigenvals()

```

```python
sy_a.eigenvects()
```

```python
from scipy import linalg as sla
```

```python
sla.eigvals(a)
```

    array([-24.14852524+0.j,  21.57934570+0.j,  12.23095798+0.j,
            14.33821487+0.j], dtype=complex64)

```python
sla.eigvalsh(a)
```

    array([-24.29197502,  10.03090763,  15.71542454,  22.54564667], dtype=float32)

```python
sla.eig(a)
```

    (array([-24.14852524+0.j,  21.57934570+0.j,  12.23095798+0.j,
             14.33821487+0.j], dtype=complex64),
     array([[ 0.02447723, -0.34770194, -0.14668337,  0.67053527],
            [ 0.02688304,  0.02018997, -0.97126281,  0.70048612],
            [-0.99910468, -0.04737921, -0.09344059,  0.05994677],
            [ 0.02163069, -0.93618959, -0.16248505, -0.23687127]], dtype=float32))

```python

```


The Black-Scholes model was first introduced by Fischer Black and Myron Scholes in 1973 in the paper "The Pricing of Options and Corporate Liabilities". Since being published, the model has become a widely used tool by investors and is still regarded as one of the best ways to determine fair prices of options.

The purpose of the model is to determine the price of a vanilla European call and put options (option that can only be exercised at the end of its maturity) based on price variation over time and assuming the asset has a lognormal distribution.

## Assumptions

To determine the price of vanilla European options, several assumptions are made:

* European options can only be exercised at expiration
* No dividends are paid during the option's life
* Market movements cannot be predicted
* The risk-free rate and volatility are constant
* Follows a lognormal distribution

## Non-Dividend Paying Black-Scholes Formula

In Black-Scholes formulas, the following parameters are defined.

* $S$, the spot price of the asset at time $t$
* $T$, the maturity of the option. Time to maturity is defined as $T - t$
* $K$, strike price of the option
* $r$, the risk-free interest rate, assumed to be constant between $t$ and $T$
* $\sigma$, volatility of underlying asset, the standard deviation of the asset returns

#### $N(d)$ is the cumulative distribution of the standard normal variable Z

$$N(d) = \frac{1}{\sqrt{2\pi}} \int_{-\infty}^d e^{-\frac{1}{2}x^2} dx$$

$C(S,t)$ is the value at time $t$ of a call option and $P(S,t)$ is the value at time $t$ of a put option.

The Black-Scholes call formula is given as:

$$C(S,t) = SN(d_1) - Ke^{-r(T - t)} N(d_2)$$

The put formula is given:

$$P(S,t) = Ke^{-r(T - t)}N(-d_2) - SN(-d_1)$$

Where:

$$d_1 = \frac{\ln \left(\frac{S}{K} \right) + \left(r + \frac{\sigma^2}{2} \right)(T - t)}{\sigma \sqrt{T - t}}$$

$$d_2 = d_1 - \sigma \sqrt{T - t} = \frac{\ln \left(\frac{S}{K} \right) + \left(r - \frac{\sigma^2}{2}\right)(T - t)}{\sigma \sqrt{T - t}}$$

## Python Implementation of Black-Scholes formula for non-dividend paying options

```python
import numpy as np
import scipy.stats as si
import sympy as sy
from sympy.stats import Normal, cdf
from sympy import init_printing
init_printing()
```

```python
def euro_vanilla_call(S, K, T, r, sigma):

    #S: spot price
    #K: strike price
    #T: time to maturity
    #r: interest rate
    #sigma: volatility of underlying asset

    d1 = (np.log(S / K) + (r + 0.5 * sigma ** 2) * T) / (sigma * np.sqrt(T))
    d2 = (np.log(S / K) + (r - 0.5 * sigma ** 2) * T) / (sigma * np.sqrt(T))

    call = (S * si.norm.cdf(d1, 0.0, 1.0) - K * np.exp(-r * T) * si.norm.cdf(d2, 0.0, 1.0))

    return call
```

```python
euro_vanilla_call(50, 100, 1, 0.05, 0.25)
```

```python
def euro_vanilla_put(S, K, T, r, sigma):

    #S: spot price
    #K: strike price
    #T: time to maturity
    #r: interest rate
    #sigma: volatility of underlying asset

    d1 = (np.log(S / K) + (r + 0.5 * sigma ** 2) * T) / (sigma * np.sqrt(T))
    d2 = (np.log(S / K) + (r - 0.5 * sigma ** 2) * T) / (sigma * np.sqrt(T))

    put = (K * np.exp(-r * T) * si.norm.cdf(-d2, 0.0, 1.0) - S * si.norm.cdf(-d1, 0.0, 1.0))

    return put
```

```python
euro_vanilla_put(50, 100, 1, 0.05, 0.25)
```

The next function can be called with 'call' or 'put' for the option parameter to calculate the desired option

```python
def euro_vanilla(S, K, T, r, sigma, option = 'call'):

    #S: spot price
    #K: strike price
    #T: time to maturity
    #r: interest rate
    #sigma: volatility of underlying asset

    d1 = (np.log(S / K) + (r + 0.5 * sigma ** 2) * T) / (sigma * np.sqrt(T))
    d2 = (np.log(S / K) + (r - 0.5 * sigma ** 2) * T) / (sigma * np.sqrt(T))

    if option == 'call':
        result = (S * si.norm.cdf(d1, 0.0, 1.0) - K * np.exp(-r * T) * si.norm.cdf(d2, 0.0, 1.0))
    if option == 'put':
        result = (K * np.exp(-r * T) * si.norm.cdf(-d2, 0.0, 1.0) - S * si.norm.cdf(-d1, 0.0, 1.0))

    return result
```

```python
euro_vanilla(50, 100, 1, 0.05, 0.25, option = 'put')
```

Sympy implementation for Exact Results

```python
def euro_call_sym(S, K, T, r, sigma):

    #S: spot price
    #K: strike price
    #T: time to maturity
    #r: interest rate
    #sigma: volatility of underlying asset

    N = Normal('x', 0.0, 1.0)

    d1 = (sy.ln(S / K) + (r + 0.5 * sigma ** 2) * T) / (sigma * sy.sqrt(T))
    d2 = (sy.ln(S / K) + (r - 0.5 * sigma ** 2) * T) / (sigma * sy.sqrt(T))

    call = (S * cdf(N)(d1) - K * sy.exp(-r * T) * cdf(N)(d2))

    return call
```

```python
euro_call_sym(50, 100, 1, 0.05, 0.25)
```

```python
def euro_put_sym(S, K, T, r, sigma):

    #S: spot price
    #K: strike price
    #T: time to maturity
    #r: interest rate
    #sigma: volatility of underlying asset

    N = systats.Normal(0.0, 1.0)

    d1 = (sy.ln(S / K) + (r + 0.5 * sigma ** 2) * T) / (sigma * sy.sqrt(T))
    d2 = (sy.ln(S / K) + (r - 0.5 * sigma ** 2) * T) / (sigma * sy.sqrt(T))

    put = (K * sy.exp(-r * T) * N.cdf(-d2) - S * N.cdf(-d1))

    return put
```

Sympy implementation of the above function that enables one to specify a call or put result.

```python
def sym_euro_vanilla(S, K, T, r, sigma, option = 'call'):

    #S: spot price
    #K: strike price
    #T: time to maturity
    #r: interest rate
    #sigma: volatility of underlying asset

    N = Normal('x', 0.0, 1.0)

    d1 = (sy.ln(S / K) + (r + 0.5 * sigma ** 2) * T) / (sigma * sy.sqrt(T))
    d2 = (sy.ln(S / K) + (r - 0.5 * sigma ** 2) * T) / (sigma * sy.sqrt(T))

    if option == 'call':
        result = (S * cdf(N)(d1) - K * sy.exp(-r * T) * cdf(N)(d2))
    if option == 'put':
        result = (K * sy.exp(-r * T) * cdf(N)(-d2) - S * cdf(N)(-d1))

    return result
```

```python
sym_euro_vanilla(50, 100, 1, 0.05, 0.25, option = 'put')
```

## Dividend Paying Black-Scholes Formula

For assets that pay dividends, the Black-Scholes formula is rather similar to the non-dividend paying asset formula; however, a new parameter $q$, is added.

* $S$, the spot price of the asset at time $t$
* $T$, the maturity of the option. Time to maturity is defined as $T - t$
* $K$, strike price of the option
* $r$, the risk-free interest rate, assumed to be constant between $t$ and $T$
* $\sigma$, volatility of underlying asset, the standard deviation of the asset returns
* $q$, the dividend rate of the asset. This is assumed to pay dividends at a continuous rate

In this case, the $q$ parameter is now included in $C(S,t)$ and $P(S,t)$.

$$C(S,t) = Se^{-q(T - t)} N(d_1) - Ke^{-r(T - t)} N(d_2)$$

$$P(S,t) = Ke^{-r(T - t)} N(-d_2) - Se^{-q(T - t)} N(-d_1)$$

Then, $d_1$ and $d_2$ are slightly modified to include the continuous dividends

$$d_1 = \frac{ln \left(\frac{S}{K} \right) + \left(r - q + \frac{\sigma^2}{2} \right)(T - t)}{\sigma \sqrt{T - t}}$$

$$d_2 = d_1 - \sigma \sqrt{T - t} = \frac{ln (\frac{S}{K}) + (r - q - \frac{\sigma^2}{2})(T - t)}{\sigma \sqrt{T - t}}$$

## Python Implementation

```python
def black_scholes_call_div(S, K, T, r, q, sigma):

    #S: spot price
    #K: strike price
    #T: time to maturity
    #r: interest rate
    #q: rate of continuous dividend paying asset
    #sigma: volatility of underlying asset

    d1 = (np.log(S / K) + (r - q + 0.5 * sigma ** 2) * T) / (sigma * np.sqrt(T))
    d2 = (np.log(S / K) + (r - q - 0.5 * sigma ** 2) * T) / (sigma * np.sqrt(T))

    call = (S * np.exp(-q * T) * si.norm.cdf(d1, 0.0, 1.0) - K * np.exp(-r * T) * si.norm.cdf(d2, 0.0, 1.0))

    return call
```

```python
def black_scholes_put_div(S, K, T, r, q, sigma):

    #S: spot price
    #K: strike price
    #T: time to maturity
    #r: interest rate
    #q: rate of continuous dividend paying asset
    #sigma: volatility of underlying asset

    d1 = (np.log(S / K) + (r - q + 0.5 * sigma ** 2) * T) / (sigma * np.sqrt(T))
    d2 = (np.log(S / K) + (r - q - 0.5 * sigma ** 2) * T) / (sigma * np.sqrt(T))

    put = (K * np.exp(-r * T) * si.norm.cdf(-d2, 0.0, 1.0) - S * np.exp(-q * T) * si.norm.cdf(-d1, 0.0, 1.0))

    return put
```

Implementation that can be used to determine the put or call option price depending on specification

```python
def euro_vanilla_dividend(S, K, T, r, q, sigma, option = 'call'):

    #S: spot price
    #K: strike price
    #T: time to maturity
    #r: interest rate
    #q: rate of continuous dividend paying asset
    #sigma: volatility of underlying asset

    d1 = (np.log(S / K) + (r - q + 0.5 * sigma ** 2) * T) / (sigma * np.sqrt(T))
    d2 = (np.log(S / K) + (r - q - 0.5 * sigma ** 2) * T) / (sigma * np.sqrt(T))

    if option == 'call':
        result = (S * np.exp(-q * T) * si.norm.cdf(d1, 0.0, 1.0) - K * np.exp(-r * T) * si.norm.cdf(d2, 0.0, 1.0))
    if option == 'put':
        result = (K * np.exp(-r * T) * si.norm.cdf(-d2, 0.0, 1.0) - S * np.exp(-q * T) * si.norm.cdf(-d1, 0.0, 1.0))

    return result
```

Sympy Implementation of Black-Scholes with Dividend-paying asset

```python
def black_scholes_call_div_sym(S, K, T, r, q, sigma):

    #S: spot price
    #K: strike price
    #T: time to maturity
    #r: interest rate
    #q: rate of continuous dividend paying asset
    #sigma: volatility of underlying asset

    N = Normal('x', 0.0, 1.0)

    d1 = (sy.ln(S / K) + (r - q + 0.5 * sigma ** 2) * T) / (sigma * sy.sqrt(T))
    d2 = (sy.ln(S / K) + (r - q - 0.5 * sigma ** 2) * T) / (sigma * sy.sqrt(T))

    call = S * sy.exp(-q * T) * cdf(N)(d1) - K * sy.exp(-r * T) * cdf(N)(d2)

    return call
```

```python
def black_scholes_call_put_sym(S, K, T, r, q, sigma):

    #S: spot price
    #K: strike price
    #T: time to maturity
    #r: interest rate
    #q: rate of continuous dividend paying asset
    #sigma: volatility of underlying asset

    N = Normal('x', 0.0, 1.0)

    d1 = (sy.ln(S / K) + (r - q + 0.5 * sigma ** 2) * T) / (sigma * sy.sqrt(T))
    d2 = (sy.ln(S / K) + (r - q - 0.5 * sigma ** 2) * T) / (sigma * sy.sqrt(T))

    put = K * sy.exp(-r * T) * cdf(N)(-d2) - S * sy.exp(-q * T) * cdf(N)(-d1)

    return put
```

Sympy implementation of pricing a European put or call option depending on specification

```python
def sym_euro_vanilla_dividend(S, K, T, r, q, sigma, option = 'call'):

    #S: spot price
    #K: strike price
    #T: time to maturity
    #r: interest rate
    #q: rate of continuous dividend paying asset
    #sigma: volatility of underlying asset

    N = Normal('x', 0.0, 1.0)

    d1 = (sy.ln(S / K) + (r - q + 0.5 * sigma ** 2) * T) / (sigma * sy.sqrt(T))
    d2 = (sy.ln(S / K) + (r - q - 0.5 * sigma ** 2) * T) / (sigma * sy.sqrt(T))

    if option == 'call':
        result = S * sy.exp(-q * T) * cdf(N)(d1) - K * sy.exp(-r * T) * cdf(N)(d2)
    if option == 'put':
        result = K * sy.exp(-r * T) * cdf(N)(-d2) - S * sy.exp(-q * T) * cdf(N)(-d1)

    return result
```

### References

[Sherbin, A. (2015). How to price and trade options: identify, analyze, and execute the best trade probabilities.
    Hoboken, NJ: John Wiley & Sons, Inc.](https://amzn.to/37ajBnM)

[Ursone, P. (2015). How to calculate options prices and their Greeks: exploring the Black Scholes model from Delta
    to Vega. Chichester: Wiley.](https://amzn.to/2UzXDrD)


# Sampyl Examples

Here I will have some examples showing how to use Sampyl. This is for version 0.2.2. Let's import it and get started. Sampyl is a Python package used to sample from probability distributions using Markov Chain Monte Carlo (MCMC). This is most useful when sampling from the posterior distribution of a Bayesian model.

Every sampler provided by Sampyl works the same way. Define $ \log P(\theta) $ as a function, then pass it to the sampler class. The class returns a sampler object, which you can then use to sample from $P(\theta)$. For samplers which use the gradient, $\nabla_{\theta} \log P(\theta)$, Sampyl uses [autograd](https://github.com/HIPS/autograd) to automatically calculate the gradients. However, you can pass in your own $\nabla_{\theta} \log P(\theta)$ functions.

Starting out simple, let's sample from a normal distribution.

```python
%matplotlib inline
%config InlineBackend.figure_format = 'retina'

import matplotlib.pyplot as plt
import sampyl as smp
from sampyl import np

# Autograd throws some warnings that are useful, but this is
# a demonstration, so I'll squelch them.
import warnings
warnings.filterwarnings('ignore')
```

A normal distribution with mean $\mu$ and variance $\sigma^2$ is defined as:

$$
P(x,\mu, \sigma) = \frac{1}{\sigma \sqrt{2 \pi}} \; \mathrm{Exp}\left( \frac{-(x - \mu)^2}{2\sigma^2} \right)
$$

For numerical stability, it is typically better to deal with log probabilities, $\log{P(\theta)}$. Then for the normal distribution with known mean and variance,

$$
\log{P(x \mid \mu, \sigma)} = -\log{\sigma} - \frac{(x - \mu)^2}{2\sigma^2}
$$

where we can drop constant terms since the MCMC samplers only require something proportional to $\log{P(\theta)}$. We can easily write this as a Python function.

```python
mu, sig = 3, 2
def logp(x):
    return  -np.log(sig) - (x - mu)**2/(2*sig**2)
```

First we'll use a Metropolis-Hastings sampler. Each sampler requires a $\log{P(\theta)}$ function and a starting state. We have included a function to calculate the *maximum a posteriori* (MAP) to find the peak of the distribution for use as the starting state. Then you call the sampler and a chain of samples is returned.

```python
start = smp.find_MAP(logp, {'x':1.})
metro = smp.Metropolis(logp, start)
chain = metro(10000, burn=2000, thin=4)
```

    Progress: [##############################] 10000 of 10000 samples

We can retrieve the chain by accessing the attributes defined by the parameter name(s) of `logp`.

```python
plt.plot(chain.x)
```

```python
_ = plt.hist(chain.x, bins=30)
_ = plt.vlines(mu, 0, 250, linestyles='--')
```

Here we have sampled from a normal distribution with a mean of 3, indicated with the dashed vertical line.

There is also a No-U-Turn Sampler (NUTS), which avoids the random-walk nature of Metropolis samplers. NUTS uses the gradient of $\log{P(\theta)}$ to make intelligent state proposals. You'll notice here that we don't pass in any information about the gradient. Instead, it is calculated automatically with [autograd](https://github.com/HIPS/autograd).

```python
nuts = smp.NUTS(logp, start)
chain = nuts(2100, burn=100)
```

    Progress: [##############################] 2100 of 2100 samples

```python
plt.plot(chain)
```

```python
_ = plt.hist(chain.x, bins=30)
_ = plt.vlines(mu, 0, 250, linestyles='--')
```

## Bayesian estimation of phone call rates

Let's try something a little more complicated. Let's say you run a business and you put an advertisement in the paper. Then, to judge the effectiveness of the ad, you want to compare the number of incoming phone calls per hour before and after the placement of the add. Then we can build a Bayesian model using a Poisson likelihood with exponential priors for $\lambda_1$ and $\lambda_2$.

\begin{align}
P(\lambda_1, \lambda_2 \mid D) &\propto P( D \mid \lambda_1, \lambda_2)\, P(\lambda_1)\, P(\lambda_2) \\
P( D \mid \lambda_1, \lambda_2) &\sim \mathrm{Poisson}(D\mid\lambda_1)\,\mathrm{Poisson}(D\mid\lambda_2) \\
P(\lambda_1) &\sim \mathrm{Exp}(1) \\
P(\lambda_2) &\sim \mathrm{Exp}(1)
\end{align}

This analysis method is known as Bayesian inference or Bayesian estimation. We want to know likely values for $\lambda_1$ and $\lambda_2$. This information is contained in the posterior distribution $P(\lambda_1, \lambda_2 \mid D)$. To infer values for $\lambda_1$ and $\lambda_2$, we can sample from the posterior using our MCMC samplers.

```python
# Fake data for the day before and after placing the ad.
# We'll make the calls increase by 2 an hour. Record data for each
# hour over two work days.
before = np.random.poisson(7, size=16)
after = np.random.poisson(9, size=16)

# Define the log-P function here
def logp(λ1, λ2):
    model = smp.Model()
    # Poisson log-likelihoods
    model.add(smp.poisson(before, rate=λ1),
              smp.poisson(after, rate=λ2))

    # Exponential log-priors for rate parameters
    model.add(smp.exponential(λ1),
              smp.exponential(λ2))

    return model()
```

```python
start = smp.find_MAP(logp, {'λ1':1., 'λ2':1.})
sampler = smp.Metropolis(logp, start)
chain = sampler(10000, burn=2000, thin=4)
```

    Progress: [##############################] 10000 of 10000 samples

Sampling returns a numpy record array which you can use to access samples by name. Variable names are taken directly from the argument list of `logp`.

```python
print(sampler.var_names)
```

    ('λ1', 'λ2')

```python
plt.plot(chain.λ1)
plt.plot(chain.λ2)
```

Now to see if there is a significant difference between the two days. We can find the difference $\delta =  \lambda_2 - \lambda_1$, then find the probability that $\delta > 0$.

```python
delta = chain.λ2 - chain.λ1
_ = plt.hist(delta, bins=30)
_ = plt.vlines(2, 0, 250, linestyle='--')
```

```python
p = np.mean(delta > 0)
effect = np.mean(delta)
CR = np.percentile(delta, (2.5, 97.5))
print("{:.3f} probability the rate of phone calls increased".format(p))
print("delta = {:.3f}, 95% CR = {{{:.3f} {:.3f}}}".format(effect, *CR))
```

    0.981 probability the rate of phone calls increased
    delta = 1.916, 95% CR = {0.154 3.746}

There true difference in rates is two per hour, marked with the dashed line. Our posterior is showing an effect, but our best estimate is that the rate increased by only one call per hour. The 95% credible region is {-0.735 2.743} which idicates that there is a 95% probability that the true effect lies with the region, as it indeed does.

We can also use NUTS to sample from the posterior.

```python
nuts = smp.NUTS(logp, start)
chain = nuts.sample(2100, burn=100)
_ = plt.plot(chain.λ1)
_ = plt.plot(chain.λ2)
```

```python
delta = chain.λ2 - chain.λ1
_ = plt.hist(delta, bins=30)
_ = plt.vlines(2, 0, 250, linestyle='--')
p = np.mean(delta > 0)
effect = np.mean(delta)
CR = np.percentile(delta, (2.5, 97.5))
print("{:.3f} probability the rate of phone calls increased".format(p))
print("delta = {:.3f}, 95% CR = {{{:.3f} {:.3f}}}".format(effect, *CR))
```

## Linear models too

When you build larger models, it would be cumbersome to have to include every parameter as an argument in the logp function. To avoid this, you can declare the size of variables when passing in the starting state.

For instance, with a linear model it would be great to pass the coefficients as one parameter. First, we'll make some fake data, then infer the coefficients.

```python
# Number of data points
N = 200
# True parameters
sigma = 1
true_B = np.array([2, 1, 4])

# Simulated features, including a constant
X = np.ones((N, len(true_B)))
X[:,1:] = np.random.rand(N, 2)*2

# Simulated outcomes with normally distributed noise
y = np.dot(X, true_B) + np.random.randn(N)*sigma

data = np.ones((N, len(true_B) + 1))
data[:, :-1] = X
data[:, -1] = y
```

```python
fig, axes = plt.subplots(figsize=(7,4),ncols=2)
for i, ax in enumerate(axes):
    ax.scatter(X[:,i+1], y)
axes[0].set_ylabel('y')
axes[0].set_xlabel('X1')
axes[1].set_xlabel('X2')
```

```python
fig.savefig('linear_model_data.png')
```

```python
# Here, β is a length 3 array of coefficients
def logp(β, sig):

    model = smp.Model()

    # Estimate from our data and coefficients
    y_hat = np.dot(X, β)

    # Add log-likelihood
    model.add(smp.normal(y, mu=y_hat, sig=sig))

    # Add prior for estimate error
    model.add(smp.exponential(sig))

    # Uniform priors on coefficients
    model.add(smp.uniform(β, lower=-100, upper=100))

    return model()
```

```python
start = smp.find_MAP(logp, {'β': np.ones(3), 'sig': 1.},
                     bounds={'β':(-5, 10), 'sig':(0.01, None)})
sampler = smp.Metropolis(logp, start)
chain = sampler(20000, burn=5000, thin=4)
```

    Progress: [##############################] 20000 of 20000 samples

```python
_ = plt.plot(chain.β)
```

And using NUTS too.

```python
start = smp.find_MAP(logp, {'β': np.ones(3), 'sig': 1.})
nuts = smp.NUTS(logp, start)
chain = nuts.sample(2100, burn=100)
```

    Progress: [##############################] 2100 of 2100 samples

```python
fig, axes = plt.subplots(figsize=(8,5), nrows=2, ncols=2)
for i, (row, param) in enumerate(zip(axes, [chain.β, chain.sig])):
    row[0].plot(param)
    row[0].set_ylabel('Sample value')
    #row[0].set_xlabel('Sample')
    row[0].set_title(['β', 'sig'][i])
    row[1].set_title(['β', 'sig'][i])
    if len(param.shape) > 1:
        for each in param.T:
            row[1].hist(each, alpha=0.8, histtype='stepfilled')
        row[1].set_yticklabels('')
        row[1].vlines([2,1,4], 0, 600, linestyles='--', alpha=0.5)

    else:
        row[1].hist(param, alpha=0.8, histtype='stepfilled')
        row[1].set_yticklabels('')
        row[1].vlines(1, 0, 600, linestyles='--', alpha=0.5)
    #row[1].set_xlabel('Sample value')
fig.tight_layout(pad=0.1, h_pad=1.5, w_pad=1)
```

```python
fig.savefig('linear_model_posterior.png')
```

## Using one logp function for both logp and gradient

You can also use one `logp` function that returns both the logp value and the gradient. To let the samplers know about this, set `grad_logp = True`. I'm also using one argument `theta` as the parameter which contains the five $\beta$ coefficients and $\sigma$.

```python
from autograd import grad
grads = [grad(logp, 0), grad(logp, 1)]
def single_logp(theta):
    b, sig = theta[:3], theta[-1]
    logp_val = logp(b, sig)
    grad_val = np.hstack([each(b, sig) for each in grads])
    return logp_val, grad_val
```

```python
start = {'theta': np.ones(4)}
nuts = smp.NUTS(single_logp, start, grad_logp=True)
chain = nuts.sample(2000, burn=1000, thin=2)
```

    Progress: [##############################] 2000 of 2000 samples

```python
_ = plt.plot(chain.theta[:, :4])
```

## Sampling in parallel

We can make use of our multicore CPUs by running chains in parallel. To do this, simply request the number of chains you want when you call `sample`: `nuts.sample(1000, n_chains=4)`. Each chain is given its own process and the OS decides how to run the processes. Typically this means that each process will run on its own core. So, if you have four cores and four chains, they will all run in parallel. But, if you have two cores, only two will run at a time.

```python
start = smp.find_MAP(logp, {'β': np.ones(3), 'sig': 1.})
nuts = smp.NUTS(logp, start)
chains = nuts.sample(1100, burn=100, n_chains=2)
```

    Progress: [##############################] 1100 of 1100 samples

```python
fig, axes = plt.subplots(figsize=(10,3), ncols=2)
for ax, chain in zip(axes, chains):
    _ = ax.plot(chain.β)
```

The future
----------

* Write a module that makes it easier to build logp functions from distributions
* Add various functions such as autocorrelation, HPD, etc.
* Plots!


# Interpolação trigonométrica: Séries de Fourier, DFT e FFT

 * Até agora usamos polinômios para aproximar e interpolar
 * Outra possibilidade: senos e cossenos
 * Extremamente versátil e poderoso
 * Usado em eqs. diferenciais, análise matemática, funções aleatórias, etc, etc, etc ...

## Polinômio trigonométrico
\begin{align}
S_N(x) = \frac{a_0}{2} +& a_1 \cos\left(\frac{2\pi x}{L_0}\right) + a_2\cos\left(\frac{4\pi x}{L_0}\right) + \cdots + a_N\cos\left(\frac{2\pi n  x}{L_0}\right) + \\
&b_1 \sin\left(\frac{2\pi x}{L_0}\right) + b_2\sin\left(\frac{4\pi x}{L_0}\right) + \cdots + b_N\sin\left(\frac{2\pi N  x}{L_0}\right)
\end{align}

Nesta equação, $T_0$ é o período da função.

O que temos é uma base $\phi_n(t)$ com $2N+1$ termos:
\begin{align}
\phi_0(x) &= \frac{1}{2}\\
\phi_n(x) &= \cos\left(\frac{2\pi n x}{L_0}\right) \qquad 0\le n \le N \\
\phi_{n+N}(x) &= \sin\left(\frac{2\pi n x}{L_0}\right) \qquad 1\le n \le N
\end{align}

## Como calcular os coeficientes $a_i$ e $b_i$???
**Ortogonalidade de senos e cossenos**

\begin{align}
\int_{-L_0/2}^{L_0/2} \cos\left(\frac{2\pi n x}{L_0}\right) \cdot \cos\left(\frac{2\pi m x}{L_0}\right) \: dx &= \left\{\begin{matrix}0 \quad n\neq m\\ \frac{L_0}{2} \quad n=m\end{matrix}\right.\\
\int_{-L_0/2}^{L_0/2} \sin\left(\frac{2\pi n x}{L_0}\right) \cdot \sin\left(\frac{2\pi m x}{L_0}\right) \: dx &= \left\{\begin{matrix}0 \quad n\neq m\\ \frac{L_0}{2} \quad n=m\end{matrix}\right.\\
\int_{-L_0/2}^{L_0/2} \cos\left(\frac{2\pi n x}{L_0}\right) \cdot \sin\left(\frac{2\pi m x}{T_0}\right) \: dx &= 0\\
\end{align}

## Como calcular os coeficientes?

\begin{align}
a_n &= \frac{2}{L_0}\int_{-L_0/2}^{L_0/2} y(x) \cdot \cos\left(\frac{2\pi n x}{L_0}\right)\: dx \\
b_m &= \frac{2}{L_0}\int_{-L_0/2}^{L_0} y(x) \cdot \sin\left(\frac{2\pi m x}{L_0}\right)\: dx \\
\end{align}

## Exemplo: $y(x) = |x|, -1 \le x \le 1$ ($L_0 = 2$)
\begin{align}
a_0 &= 2\int_0^1 x\:dx \qquad a_0 = 1\\
a_n &= 2\int_0^1 \cos\left(\frac{2\pi n x}{2}\right)\:dx \qquad a_n = \frac{2}{\pi^2 n^2}\cdot\left[(-1)^n - 1\right] \\
b_n &= 0
\end{align}

## Usando o SymPy

```julia
using SymPy
x = symbols("x", real=true)
n = symbols("n", integer=true, positive=true)
a0f  =  2 * integrate(x, (x, 0, 1))
anf = 2 * integrate(x * cos(2PI*n*x/2), (x, 0, 1))
```

```julia
function abst(t, n)
    x = 1.0/2
    for i in 1:2:n
        ai = 2/(π*i)^2 * ((-1)^i - 1)
        x += ai * cos(π*i*t)
    end
    return x
end
```

```julia
using PyPlot
```

```julia
xx = -1:0.002:1; yy = abs.(xx); y1 = abst.(xx, 1); y3 = abst.(xx, 3); y5 = abst.(xx, 5); y7 = abst.(xx, 7);
plot(xx, yy, "b:", label="Exato"); plot(xx, y1, label="n = 1"); plot(xx, y3, label="n = 3");
plot(xx, y5, label="n = 5"); plot(xx, y7, label="n = 7"); legend()
```

## Vamos generalizar?

Imagine que queremos aproximar $u(x)$ usando uma base ortogonal $\phi_n(x)$ com $1 \le n \le N$ no domínio $a \le x \le b$:
$$
u(x) \approx u^\delta(x) = \sum_{n=1}^N \hat{u}_n \phi_n(x)
$$
O erro da aproximação vale:
$$
\varepsilon(x) = u^\delta(x) - (x) \quad\longrightarrow\quad \varepsilon^2 = \int_a^b w(x)\left[ u^\delta(x) - u(x) \right]^2\:dx
$$

Nesta equação, $w(x)$ é uma função de peso que agora vamos considerar constante e igual a 1.

## Mínimos quadrados.
$$
\varepsilon^2 = f(\hat{u}_1, \ldots, \hat{u}_N)
$$
Para minimizar o erro, fazemos como fizemos na aula anterior:
$$
\frac{\partial \varepsilon^2}{\partial \hat{u}_k} = 0 \quad 1 \le k \le N
$$

Com isso chegamos ao seguinte sistema de equações lineares:
$$
\sum_{j=1}^N \hat{u}_j \int_a^b w(x)\phi_n(x)\phi_k(x)\:dx = \int_a^b w(x)\hat{u}_k u(x)\:dx \qquad 1\le k \le N
$$

## Base ortogonal:

$$
\int_a^b w(x) \phi_j(x) \phi_k(x) \:dx =
\left\{\begin{matrix} 0 & \quad &  se\: j\ne k \\ \gamma(k) & \quad & se\: j=k\end{matrix}\right.
$$
Quando $\gamma(k) = 1$ para $1\le k \le N$, a base é ortonormal.

Para a função de peso $w(x)=1$ e a base formada com senos e cossenos, recuperamos as equações para $a_n$ e $b_n$ acima.

**Sempre que possível, use bases ortogonais**

Quando não for possível, ainda ajuda usar bases parcialmente ortogonais.

## Mínimos quadrados discreto

Neste caso, com uma base $\phi_n(x)$ com $1\le n \le N$ e com a função $u(x)$ conhecida nos pontos $x_k$, $1 \le k \le Q$, podemos usar o método dos mínimos quadrados:
$$
\varepsilon^2 = \sum_{k=1}^Q w_k\left[u^\delta(x_k) - u(x_k)\right]^2 = \sum_{k=1}^Q w_k\left[\left(\sum_{n=1}^N \hat{u}_n \phi_n(x)\right) - u_k\right]^2
$$

Chegamos ao sistema apresentado na aula anterior:

$$
\left(
\begin{matrix}
\sum_{k=1}^Q w_k\phi_1(x_k)\cdot\phi_1(x_k) &
\cdots &
\sum_{k=1}^Q w_k\phi_1(x_k)\cdot\phi_N(x_k) \\
\sum_{k=1}^Q w_k\phi_2(x_k)\cdot\phi_1(x_k) &
\cdots &
\sum_{k=1}^Q w_k\phi_2(x_k)\cdot\phi_N(x_k) \\
\vdots & \ddots & \vdots \\
\sum_{k=1}^Q w_k\phi_N(x_k)\cdot\phi_1(x_k) &
\cdots &
\sum_{k=1}^Q w_k\phi_N(x_k)\cdot\phi_N(x_k) \\
\end{matrix}\right)
\cdot
\left(\begin{matrix} \hat{u}_1 \\ \hat{u}_2 \\ \vdots \\ \hat{u}_N \end{matrix}\right)
=
\left(\begin{matrix} \sum_{k=1}^Q w_k u_k \phi_1(x_k) \\ \sum_{k=1}^Q w_k u_k \phi_2(x_k) \\  \vdots \\ \sum_{k=1}^Q w_k u_k \phi_N(x_k)\end{matrix}\right)
$$

Aqui introduzimos a possibilidade de haver um peso $w_k$ para cada ponto.

## Voltando para os polinômios trigonométricos...

### Ortogonalidade discreta
Já vimos que a base dos polinômios trigonométricos é ortogonal, de modo que para $k\ne n$,
$$
\int_{-L_0/2}^{L_0/2} \phi_n(x) \cdot \phi_k(x) \: dx = 0
$$
Mas podemos aproximar esta integral, chamando $\Delta x = L_0 / Q$
$$
\int_{-L_0/2}^{L_0/2} \phi_n(x) \cdot \phi_k(x) \: dx \approx \Delta x\cdot \sum_{i=0}^Q \phi_n(x_i)\cdot \phi_k(x_i)  \approx 0 \qquad x_i = -\frac{L_0}{2} + i\cdot\Delta x\quad i = 0, \ldots, Q-1
$$

## Ortogonalidade discreta
Admitindo que temos $Q$ pontos $x_j = j\cdot \Delta x$ para $0\le j\le Q$ e $\Delta x = L_0/Q$, demonstra-se que

$$
\sum_{j=0}^{Q-1} \cos\left(\frac{2\pi n x_j}{L_0}\right) = 0 \qquad
\sum_{j=0}^{Q-1} \sin\left(\frac{2\pi n x_j}{L_0}\right) = 0
$$
quando $n$ é multiplo de $Q$. Por outro lado

$$
\sum_{j=0}^{Q-1} \left[\cos\left(\frac{2\pi n x_j}{L_0}\right)\right]^2 = \frac{Q}{2} \qquad
\sum_{j=0}^{Q-1} \left[\sin\left(\frac{2\pi n x_j}{L_0}\right)\right]^2 = \frac{Q}{2}
$$
quando $n$ não é múltiplo de $Q/2$

# Ortogonalidade discreta
Lembrando que
\begin{align}
    2\cos\theta\cos\phi &= \cos(\theta-\phi) + \cos(\theta+\phi)\\
    2\cos\theta\sin\phi &= \sin(\theta+\phi) - \sin(\theta-\phi)\\
\end{align}

Chega-se à relação de ortogonalidade desejada:
$$
\sum_{i=0}^Q \phi_n(x_i)\cdot \phi_k(x_i)=0 \qquad k\ne n,k+n \text{ e } k-n \text{ não são múltiplos de}\: Q/2
$$

## Transformada de Fourier real discreta

Com as relações de ortogonalidade, chega-se à seguinte relação para os coeficientes $a_n$ e $b_n$:

\begin{align}
a_n &= \frac{2}{Q}\cdot\sum_{j=0}^{Q-1} y_j\cos\left(\frac{2\pi n x_j}{L_0}\right) \qquad n=0, 1, \ldots, N\\
b_n &= \frac{2}{Q}\cdot\sum_{j=0}^{Q-1} y_j\sin\left(\frac{2\pi n x_j}{L_0}\right) \qquad n=1, \ldots, N
\end{align}

## Outra abordagem para a transformada de Fourier real discreta
\begin{align}
a_n &= \frac{2}{L_0}\int_{-L_0/2}^{L_0/2} y(x) \cdot \cos\left(\frac{2\pi n x}{L_0}\right)\: dx \\
b_m &= \frac{2}{L_0}\int_{-L_0/2}^{L_0} y(x) \cdot \sin\left(\frac{2\pi m x}{L_0}\right)\: dx \\
\end{align}

Aproximando as integrais:
\begin{align}
a_n &= \frac{2}{L_0}\int_{-L_0/2}^{L_0/2} y(x) \cdot \cos\left(\frac{2\pi n x}{L_0}\right)\: dx \approx \frac{2}{Q\cdot \Delta x} \sum_{j=0}^{Q-1} y(x_j) \cos\left(\frac{2\pi n x_j}{L_0}\right) \cdot \Delta x =\\
 & =  \frac{2}{Q} \sum_{j=0}^{Q-1} y_j \cos\left(\frac{2\pi n x_j}{L_0}\right)
\end{align}

Onde $\Delta x = L_0 / Q$

E assim obtemos o mesmo resultado que no slide anterior!

## Exemplo: $y(x) = |x|, -1 \le x \le 1$ ($L_0 = 2$)

```julia
function fourierseries(x, L0, a0, a, b)
    y = a0/2
    N = length(a)
    for i = 1:N
        y += a[i] * cos(2π*i*x/L0)
        y += b[i] * sin(2π*i*x/L0)
    end
    return y
end

```

```julia
L0 = 2.0
Q = 20
N = 8

x = range(-1.0, 1.0, length=Q+1)[1:Q]
y = abs.(x)

a0 = 2/Q * sum(y)
a = 2/Q * [sum(y .* cos.(2π .* n .* x ./ L0)) for n = 1:N]
b = 2/Q * [sum(y .* sin.(2π .* n .* x ./ L0)) for n = 1:N];
```

```julia
# y(x) = |x|, -1 \le x \le 1$ ($L_0 = 2$)
xx = -1:0.005:1
yy = abs.(xx)
yy1 = fourierseries.(xx, L0, a0, Ref(a), Ref(b))
plot(xx, yy)
plot(xx, yy1);
```

## Problemas digitais:

O que nós temos:
$$
y_j, \Delta x, \quad\longrightarrow\quad x_{j+1}-x_j = \Delta x =\frac{L_0}{Q} \qquad 0\le j < Q
$$
não temos os valores de $x_n$ exatos. Então podemos assumir $x_j = j \Delta x$:
\begin{align}
 a_n &= \frac{2}{Q} \sum_{j=0}^{Q-1} y_j \cos\left(\frac{2\pi n x_j}{L_0}\right) =
 \frac{2}{Q} \sum_{j=0}^{Q-1} y_j \cos\left(\frac{2\pi n j \Delta x}{Q\cdot\Delta x}\right) =
 \frac{2}{Q} \sum_{j=0}^{Q-1} y_j \cos\left(\frac{2\pi n j}{Q}\right) \\
 b_n &= \frac{2}{Q} \sum_{j=0}^{Q-1} y_j \sin\left(\frac{2\pi n x_j}{L_0}\right) =
 \frac{2}{Q} \sum_{j=0}^{Q-1} y_j \sin\left(\frac{2\pi n j}{Q}\right)
\end{align}

# Forma complexa da série de Fourier

$$
c_n = \left\{ \begin{matrix}
\frac{a_0}{2} & n = 0 \\
\frac{1}{2}\left(a_n - i b_n\right) & n > 0\\
\frac{1}{2}\left(a_n + i b_n\right) & n < 0
\end{matrix}\right.
$$
$$
f(x) = \sum_{n=-N}^N c_n \exp\left(\frac{2\pi i n x}{L_0} \right) \qquad
c_n = \frac{1}{L_0}\int_0^{L_0} f(x) \exp\left(-\frac{2\pi i n x}{L_0} \right)\:dx
$$
Se $f(x)$ for uma função real, para $n$ positivo:
$$
\begin{matrix}
c_n + c_{-n} = a_n\\
c_n - c_{-n} = -i b_n
\end{matrix}
\longrightarrow
c_n = c^*_{-n}
$$

## DFT - Transformada de Fourier discreta

$$
c_n = \frac{1}{L_0}\int_0^{L_0} f(x) \exp\left(-\frac{2\pi i\cdot n x}{L_0} \right)\:dx \approx \frac{1}{\Delta x \cdot Q}\sum_{j=0}^{Q-1} y_j \exp\left(-\frac{2\pi i \cdot n j\Delta x}{Q\cdot \Delta x} \right)\cdot\Delta x
$$

Portanto,
$$
c_n = \frac{1}{Q}\sum_{j=0}^{Q-1} y_j \exp\left(-\frac{2\pi i\cdot n j}{Q} \right)
$$

# DFT - Discrete Fourier Transform

$$
c_n = X_n = \frac{1}{N}\sum_{k=0}^{N-1} x_k \exp\left(-\frac{2\pi n k}{N} \right)
$$

Chamando
$$
w_k = w^k = \exp\left(-\frac{2\pi k}{N}\right)
$$
Repare que isto pode ser escrito como uma multiplicação de matrizes:

$$
\left(\begin{matrix} X_0 \\ X_1 \\ \vdots \\ X_{N-1}\end{matrix} \right)
=
\left(\begin{matrix} 1 & 1 & \cdots & 1\\
1 & w & \cdots & w^{N-1} \\
\vdots & \vdots & \ddots &\vdots \\
1 & w^N & \cdots & w^{(N-1)^2}\\
\end{matrix}\right)
\cdot
\left(\begin{matrix} x_0 \\ x_1 \\ \vdots \\ x_{N-1}\end{matrix} \right)
$$

## FFT - Transformada Rápida de Fourier

Um algorítmo rápido para
$$
\sum_{j=0}^{Q-1} y_j \exp\left(-\frac{2\pi i\cdot n j}{Q} \right)
$$

Se $0\le n\le Q$, o custo computacional é $\mathcal{O}(Q^2)$.

FFT: custo cai para $\mathcal{O}(Q\cdot\log Q)$

Originalmente, só funcionava com $Q$ igual a potências de 2.

```julia
Q = [2, 3, 5, 10, 15, 20, 40, 60, 80, 100, 150, 200, 300, 500, 700, 1000, 2000, 5000, 10000,
20000, 30000, 50000, 100000, 200000, 500000, 1000000]
t1 = Q
t2 = Q.^2
tlog = Q .* log2.(Q);
loglog(Q, t1, label="\$\\mathcal{O}(Q)\$")
plot(Q, t2, label="\$\\mathcal{O}(Q^2)\$")
plot(Q, tlog, label="\$\\mathcal{O}(Q\\cdot\\log_2 Q)\$")
legend()

```

## FFTW - Fastest Fourier Transform of the West

```julia
## Pkg.add("FFTW")
using FFTW
```

```julia
L₀ = 1.0; Q = 4; freq=1
x = range(0.0, L₀, length=Q+1)[1:Q]
y = sin.(freq.*2π.*x); f = (0:Q-1) / L₀
Y = 1/Q * fft(y)
```

## Exemplos

```julia
L₀ = 1.0; Q = 16; freq=1
x = range(0.0, L₀, length=Q+1)[1:Q]
y = sin.(freq.*2π.*x); f = (0:Q-1) / L₀
Y = 1/Q * fft(y)
plot(f, imag.(Y), "ro", label="Parte Imaginária")
plot(f, real.(Y), "bx", label="Parte Real")
xlabel("Frequência (Hz)"); axvline(freq, linestyle=":")
annotate("???", (f[Q], imag(Y[Q])))
legend()
```

```julia
L₀ = 1.0; Q = 8; freq=1
x = range(0.0, L₀, length=Q+1)[1:Q]; f = (0:Q-1) / L₀

y = cos.(freq.*2π.*x) + 2*sin.(2*freq.*2π.*x)

Y = 1/Q * fft(y)
plot(f, imag.(Y), "ro", label="Parte Imaginária")
plot(f, real.(Y), "bx", label="Parte Real")
xlabel("Frequência (Hz)"); axvline(freq, linestyle=":")
legend()
```

```julia
fftfreq(Q) * Q
```

## A DFT é periódica (aliasing)!

Sejam os pontos $y_j$, $0 \le j < N$:

$$
Y_k = \frac{1}{N}\sum_{j=0}^{N-1} y_j\exp\left(-\frac{2\pi i j k}{N}\right)
$$
$$
Y_{N+k} = \frac{1}{N}\sum_{j=0}^{N-1} y_j\exp\left[-\frac{2\pi i j (k+N)}{N}\right] =
\frac{1}{N}\sum_{j=0}^{N-1} y_je^{-2\pi i j}\exp\left(-\frac{2\pi i j k}{N}\right)
$$
$$
Y_{N+k} = \frac{1}{N}\sum_{j=0}^{N-1} y_j\exp\left(-\frac{2\pi i j k}{N}\right) = Y_k
$$

## Exemplo de Aliasing

```julia
ncyc = 4; Q = 13; T₀ = 1; g(t) = sin(3*2π * t)
t = range(0, ncyc, length=Q+1); x = g.(t)
t0 = 0:0.01:ncyc; x0 = g.(t0)
plot(t0, x0); plot(t, x, "ro-")

```

```julia

```

## O que acontece com frequências negativas (quando $y_j$ é real)?

E se o $y_j$ forem reais?
$$
Y_{-k} = \frac{1}{N}\sum_{j=0}^{N-1} y_j\exp\left(\frac{2\pi i j k}{N}\right) = Y^*_k
$$

Mas lembre-se que $Y$ também é periódico:

$$
Y_{-k} = Y_{N-k} = Y^*_k
$$

```julia
L₀ = 1.0; Q = 8; freq=5
x = range(0.0, L₀, length=Q+1)[1:Q]
y = sin.(freq.*2π.*x); f = (0:Q-1) / L₀
Y = 1/Q * fft(y)
plot(f, imag.(Y), "ro", label="Parte Imaginária")
plot(f, real.(Y), "bx", label="Parte Real")
xlabel("Frequência (Hz)"); axvline(freq, linestyle=":")
annotate("???", (f[Q], imag(Y[Q])))
legend()
```

## Qual o número de frequencias relevantes?

 * Q números reais
 * `fft` gera Q números complexos - 2 $\times$ vezes mais informação
 * A frequência 0 sempre vai corresponder a um número real (a média)!

### Q é um número ímpar
  * $a_0$
  * $(Q-1)/2\:\longrightarrow\:a_n$
  * $(Q-1)/2\:\longrightarrow\:b_n$

### N é um número par
 * $a_0$
 * $Q/2\:\longrightarrow\:a_n$
 * $Q/2\:\longrightarrow\:b_n$ mas $b_{1+Q/2} = 0$

$$
N_\text{relevante} = \frac{Q}{2} + 1
$$

```julia
#nfftutil(N) = (N % 2 == 0) ? (div(N,2) + 1) : div(N+1,2)
nfftutil(N) = div(N,2) + 1

```

```julia
div(3,2)
3 % 2
```

```julia
nfftutil(3)
```

```julia
nfftutil(4)
```

```julia
nfftutil(5)
```

```julia
nfftutil(6)
```

## `rfft`: especializado para números reais

```julia
Q = 6
x = rand(Q) .+ 10
println(sum(x))
X = rfft(x)

```

```julia
X = fft(x)
```

## Interpolação trigonométrica

```julia
struct TrigInterp
    Q::Int
    dx::Float64
    c::Vector{ComplexF64}
end
TrigInterp(x::Vector{Float64}, dx) = TrigInterp(length(x), dx, rfft(x))

function interp(f::TrigInterp, x)
    y = real(f.c[1])/2
    Q = f.Q
    P = Q * f.dx
    N = length(f.c)
    for k in 1:N-1
        c = f.c[k+1]
        y += real(c) * cos(2π*k*x/P) - imag(c) * sin(2π*k*x)
    end

    return 2y/Q
end

(f::TrigInterp)(x) = interp(f, x)
```

## Exemplo

```julia
Q = 7; L₀ = 1.0;
x0 = range(0, L₀, length=Q+1)[1:Q]
f = x->(2*(sin(2π*x)) + 1.0 + cos(2π*x))
y0 = f.(x0)
Y = TrigInterp(y0, L₀/Q)
x = 0:0.01:L₀; y = Y.(x);
plot(x, f.(x), label="Exato")
plot(x, y, "k:", label="Interpolado")
plot(x0, y0, "ro", label="Pontos")
legend()

```

## E usando fft (não rfft)?

```julia
struct TrigInterp0
    Q::Int
    dx::Float64
    c::Vector{ComplexF64}
end
TrigInterp0(x::Vector{Float64}, dx) = TrigInterp0(length(x), dx, fft(x))

function interp(f::TrigInterp0, x)
    y = real(f.c[1])/2
    Q = f.Q
    P = Q * f.dx
    N = nfftutil(length(f.c))
    for k in 1:N-1
        c = f.c[k+1]
        y += real(c) * cos(2π*k*x/P) - imag(c) * sin(2π*k*x)
    end

    return 2y/Q
end

(f::TrigInterp0)(x) = interp(f, x)
```

```julia
Y = TrigInterp0(y0, L₀/Q)
x = 0:0.01:L₀; y = Y.(x);
plot(x, f.(x), label="Exato")
plot(x, y, "k:", label="Interpolado")
plot(x0, y0, "ro", label="Pontos")
legend()

```

## Transformação inversa

```julia
x = rand(4)
X = fft(x)
x1 = ifft(X)
x1 .- x
```

```julia
Y = rfft(x)
x2 = irfft(Y, 4)
x2 .- x
```

## Resample

```julia

function myresample(y, Qout)
    Qin = length(y)
    Nin = Qin ÷ 2 + 1
    Nout = Qout ÷ 2 + 1

    Yin = rfft(y)
    Yout = zeros(ComplexF64, Nout)
    nmin = min(Nout, Nin)
    for i in 1:nmin
        Yout[i] = Qout/Qin * Yin[i]
    end

    if Qout > Qin  # interpolação
        if iseven(Qin)
            Yout[Nin] /= 2
        end
    elseif Qout < Qin
        if iseven(Qin)
            Yout[Nout] = 2*real(Yout[Nout])
        end
    end

    return irfft(Yout, Qout)
end

```

## Usando `myresample`

```julia
Q = 8; L₀ = 1.0;
x0 = range(0, L₀, length=Q+1)[1:Q]; x = 0:0.01:L₀
f = x->(2*sin(2π*x) + 1.0 + cos(2π*x))
y0 = f.(x0)
Qout = 10
y1 = myresample(y0, Qout);
x1 = range(0, L₀, length=Qout+1)[1:Qout]
plot(x, f.(x), label="Exato")
plot(x1, y1, "k:", label="Interpolado")
plot(x0, y0, "ro", label="Pontos")
legend()

```

## Outros formatos

```julia
x = rand(4)
X1 = fft(x)
X2 = copy(X)
fft!(X2)
```

O mesmo para `ifft`, `rfft`, etc!

## Lembra que a DFT era uma multiplicação de matrizes?

```julia
x = rand(4)
P = plan_fft(x, flags=FFTW.ESTIMATE)

X1 = P * x
```

```julia
fft(x)
```

## Transformada inversa

```julia
P \ X1
```

```julia
ifft(X1)
```

## Outros tipos de transformadas

 * Cosseno: `dct`, `idct`
 * Mais genérica: `r2r` (ver <www.fftw.org>)
 * Tudo o que foi feito até aqui, vale para problemas multidimensionais!

## Conclusões

Algumas questões

 * Será que esta série converge para a função original?
 * Em que condições esta série converge?
 * Quão rápido se dá esta convergência?
 * O que acontece quando $N\longrightarrow\infty$?

Respostas genéricas
 * A série converge para condições bem amplas, inclusive se houver discontinuidades na função.
 * Esta convergência se dá para $N\longrightarrow\infty$. Para valor de $N$ finito, as coisas são mais complicadas (como veremos adiante).
 * Para funções periódicas e suaves, não existe aproximação melhor!!!

# Problemas

## Problema 1

Seja a seguinte função definida no domínio $-1 < x \le 1$:

$$
y(x) = \left\{\begin{matrix} -1 & x < 0 \\ 1 & x > 0\\ 0 & x=0\end{matrix}\right.
$$

 1. Ache os coeficientes $a_n$ e $b_n$ da série de Fourier
 2. Plote a série de Fourier para diferentes números de termos da série de Fourier. Percebe algo estranho?
 3. Qual a relação desta função com a função $y = |x|$ que estudamos anteriormente?
 4. Escolha diferentes números de pontos e interpole esta função nestes pontos

## Problema 2

Verifique a ortogonalidade discreta da base trigonométrica, ou seja,

$$
\sum_{i=0}^Q \phi_n(x_i)\cdot \phi_k(x_i)=0 \qquad k\ne n, k+n \text{ e } k-n \text{ não são múltiplos de}\: Q/2
$$

para

\begin{align}
\phi_0(x) &= \frac{1}{2}\\
\phi_n(x) &= \cos\left(\frac{2\pi n x}{L_0}\right) \qquad 0\le n \le N \\
\phi_{n+N}(x) &= \sin\left(\frac{2\pi n x}{L_0}\right) \qquad 1\le n \le N
\end{align}

## Problema 3

Nos exemplos iniciais usando a DFT, ao se escolher os pontos, foi utilizado o seguinte código:
```julia
L₀ = 1.0; Q = 16; freq=1
x = range(0.0, L₀, length=Q+1)[1:Q]
```
O que acontece se utilzarmos a expressão mais simples a seguir?
```julia
x = range(0.0, L₀, length=Q)
```
O que está acontecendo? Você consegue explicar?

```julia
L₀ = 1.0; Q = 16; freq=1;
x = range(0.0, L₀, length=Q)
y = sin.(freq.*2π.*x); f = (0:Q-1) / L₀
Y = 1/Q * fft(y);
#plot(f, real.(Y), "xb")
#plot(f, imag.(Y), "ro")
```

## Problema 4

Implemente a DFT usando a definição. Verifique como aumenta o custo computacional quando o número de pontos aumenta.

## Problema 5

(Desafio) procure na net ou em algum livro como implementar a FFT. Tente!

## Problema 6

A transformada de Fourier está diretamente relacionada com a série de Fourier e é dada por:

$$
X(\omega) = \frac{1}{2\pi} \int_{-\infty}^\infty x(t)e^{-i\omega t}\: dt
$$

 1. Calcula a transformada de Fourier para a função
  $$
  x(t) = \left\{\begin{matrix} 1 & -1 < x < 1 \\ 0 & x \ge 1\end{matrix}\right.
  $$
 2. Discretize a função e use a FFT para calcular a transformada.
 3. Tente verificar o que acontece se você adiciona zeros à esquerda e à direita da função discretizada.

## Problema 7

Faça alguns exemplos do pacote `ApproxFun` (<https://github.com/JuliaApproximation/ApproxFun.jl>)

```julia

```


# Intro
$\DeclareMathOperator*{\argmin}{argmin}
\newcommand{\ffrac}{\displaystyle \frac}
\newcommand{\Tran}[1]{{#1}^{\mathrm{T}}}
\newcommand{\d}[1]{\displaystyle{#1}}
\newcommand{\EE}[2][\,\!]{\mathbb{E}_{#1}\left[#2\right]}
\newcommand{\dd}{\mathrm{d}}
\newcommand{\Var}[2][\,\!]{\mathrm{Var}_{#1}\left[#2\right]}
\newcommand{\using}[1]{\stackrel{\mathrm{#1}}{=}}
\newcommand{\I}[1]{\mathrm{I}\left( #1 \right)}
\newcommand{\N}[1]{\mathrm{N} \left( #1 \right)}
\newcommand{\space}{\text{ }}
\newcommand{\QQQ}{\boxed{?\:}}$

# Sample Space and Events
Suppose an experiment whose outcome is not predictable in advance, only the set of all possible outcomes is known. We call this set: ***sample space***, $S$, of this experiment.

And an ***Event*** is any subset of this sample space, denoted as $E$. The event $E$ **occurs** when the outcome lies in $E$. And following that we can define the Union, Intersection, and the ***null event***, $\varnothing$. We call two sets are ***mutually exclusive*** when their intersection is a null event.

Finally we define the ***complement***. For any event $E$ we define the new event $E^c$, referred to as the complement of $E$, to *consists of all outcomes in the sample sapce $S$ that are not in $E$*.

# Probability Defined on Events
Consider an experiment, with sample sapce $S$, and for each event $E$, we assume that a **number** $P(E)$ is defined and satisfies the following these conditions:

- $0 \leq P(E) \leq 1$
- $P(S) = 1$
- For **any sequence** of events $E_1, E_2, \dots$ that are mutually exclusive, that is events for which $E_nE_m = E_n \cap E_m = \varnothing$ when $n \neq m$, then
$\d{P\left(\bigcup_{n=1}^{\infty}E_n\right) = \sum_{n=1}^{\infty} P(E_n)} $

$\QQQ$Can this **any sequence** be limited?

We refer to $P(E)$ as the ***probability*** of the event $E$.

$Remark$

>These probabilities have a nice intuitive property, that if our experiment is repeated over and over again $wp1$, the proportion of time that event $E$ occurs will just be $P(E)$.

Other conclusions

- $1 = P(S) = P(E \cup E^c) = P(E) + P(E^c)$ or $P(E^c) = 1 - P(E)$
- $P(\varnothing) = 1 - P(S) = 0$
- $P(E) + P(F) = P(E \cup F) + P(EF)$, and when they're mutually exclusive, we have $P(E) + P(F) = P(E \cup F)$
- $(EG \cup FG ) = \cdots = (E \cup F)G $
- $P(E \cup F \cup G) = P((E\cup F)\cup G) = P(E \cup F) + P(G) - P((E\cup F)G)\\
\Rightarrow P(E \cup F \cup G) = P(E) + P(F) + P(G) - P(EF) - P(FG) - P(GF) + P(EFG)$
- and by **induction** that for any $\mathbf{n}$ events $E_1,E_2, \dots, E_n$, we have:$\\[0.5em]$
$$\begin{align}
P(E_1 \cup E_2 \cup \cdots \cup E_n) &= \sum_{i}P(E_i)  \sum_{i < j}P(E_iE_j) + \sum_{i<j<k}P(E_iE_jE_k) \\
&\;\;\; - \sum_{i<j<k<l}P(E_iE_jE_kE_l) + \dots \\
&\;\;\; + (-1)^{n+1} P(E_1E_2E_3\cdots E_n)
\end{align}$$

# Conditional Probabilities

***Conditional Probability*** for event $E$ given that event $F$ has occured is denoted by $P(E\mid F)$, and the formula for this is

$$P(E \mid F) = \ffrac{P(EF)} {P(F)}$$

$Remark$

This is well defined when $P(F)>0$, hence $P(E \mid F)$ is only defined when $P(F)>0$.

# Independent Events

Event $E$ and $F$ are ***independent*** if $P(EF) = P(E)P(F)$. And an direct result is $P(E \mid F) = P(E)$, otherwise they are ***dependent***. And to prove independency, we need both $P(EF) = P(E)P(F)$ and $P(E \mid F) = P(E)$, then that will work.

An extended definition, for $n$ events $E_1, \dots, E_n$. If for every subset $E_1', \dots, E_r'$, with $r \leq n$, of these events, we always have $P(E_1' \dots E_r') = P(E_1')\dots P(E_r')$.

$Remark$

***Pairwise independence*** CANNOT ensure independence. And to distinguish these two, we call that ***jointly independent***.

**e.g.**
$r$ players each initially having $n_i$ units, $n_i > 0$, $i = 1, \dots, r$. For each stage two players are chosen to play, with the winner win $1$ unit from the loser. Any player whose fortune drops to $0$ is eliminated and game continues until one single player has all $n \equiv \sum n_i$ units. Assuming that the results of successive games are independent and that each game is equally likely to be won by either of its two players. Find the probability that player $i$ is the victor!

> First we suppose there're totoally $n$ players with each player initially having only $1$ unit and others are same with before.
>
>Now because this is the same for all players, it follows that each player has the same chance of being the victor. Consequently, each player has player probability $1/n$ of being the victor.
>
>Now suppose these players are divided into $r$ teams, with team $i$ containing $n_i$ players. Then it's easy to see that the probability that the victor is a member of team $i$ is $n_i/n$. And this is same for the previous problem.

***

Suppose that a sequence of experiments, each of which results in either a $1$ or a $0$, is to be performed. Let $E_i$, $i \geq 1$ denote the event that the $i\text{-th}$ experiment results in a success. We call this sequence of experiments consists of **independent trials**, if for all $i_1, i_2, \dots, i_n$, this holds

$\;\;\;\;\d{P(E_{i_1} E_{i_2} \cdots E_{i_n}) = \prod_{j=1}^{n}P(E_{i_j})}$

# Bayes' Formula

Let $E$ and $F$ be events. We may express $E$ as $E = EF \cup EF^c$ because in order for a point to be in $E$, it must either in both $E$ and $F$ or it must be in $E$ and not in $F$.

And since $EF$ and $EF^c$ are mutually exclusive, we have that

$$\begin{align}
P(E) &= P(EF) + P(EF^c) \\
&= P(E \mid F) P(F) + P(E\mid F^c) P(F^c) \\
&= P(E \mid F) P(F) + P(E\mid F^c) (1-P(F))
\end{align}$$

which implies that $P(E)$ can be seen as a **weighted average** of

- the conditional probability of $E$ given that $F$ has occurred
- the conditional probability of $E$ given that $F$ has not occured.

and each is given as much weight as the event on which it is contioned has of occurring.

**e.g.**
First toss with results $H$ or $T$, each with probability $1/2$. Then for result $H$, get $W$ with probability $2/9$ and $B$ with probability $7/9$ and for result $T$, get $W$ with probability $5/11$ and $B$ with probability $6/11$. Now given $W$, what's the conditional probability of $H$?

>$\d{P(H\mid W) = \frac{P(HW)} {P(W)} = \frac{P(HW)} {P(W\mid H)P(H) + P(W \mid T)P(T)}} = \frac{\frac{2} {9} \frac{1} {2}} {\frac{2} {9} \frac{1} {2} + \frac{5} {11}\frac{1} {2}} = \frac{22} {67}$

***

Here's the generalized method. Suppose that $F_1, F_2, \dots, F_n$ are mutually exclusive events such that $\bigcup_{i=1}^{n}F_i = S$. In other words, exactly one of the events $F_1, F_2, \dots, F_n$ will occur. Then we write:

$$E = \bigcup_{i=1}^{n}EF_i$$

and using the fact that events $EF_i$ for $i = 1, 2, \dots, n$ are also mutually exclusive, we obtain that:

$$
\begin{align}
P(E) &= \sum_{i=1}^{n} P(EF_i) \\
&= \sum_{i=1}^{n} P(E \mid F_i) P(F_i)
\end{align}
$$

Then we have the formula, the ***Bayes' formula***:

$$\boxed{
P(F_j\mid E) = \ffrac{P(EF_j)} {P(E)} = \frac{P(E \mid F_j)P(F_j)} {\sum_{i=1}^{n} P(E \mid F_i) P(F_i)}}
$$

# Exercises

**32**
Suppose $n$ men throw their hats in the center of the room. Each men then randomly selects a hat. Show that the probability that **none** of these $n$ men selects his own hat is

$$\frac{1} {2!} - \frac{1} {3!} + \frac{1} {4!} - + \cdots \frac{(-1)^n} {n!}$$

>Let $E_i$ denote the event that person $i$ selects his own hat, then we have the finanl probability equals to
>
>$$\begin{align}
P &= 1 - P(E_1 \cup E_2 \cup \cdots \cup E_n) \\
&= 1 - \left[ \sum_{i} P(E_{i}) - \sum_{i < j}P(E_iE_j) + \cdots + (-1)^{n+1}P(E_1E_2\cdots E_n) \right] \\
\end{align}$$
>
>After applying the formula before, the question facing now is what is $P(E_{i_1}\cdots E_{i_{h}})$. Here's how we calculate them
>
>$$\begin{align}
\sum_{i_1 < \cdots < i_h} P(E_{i_1}\cdots E_{i_{h}}) &= \sum_{i_1 < \cdots < i_h} \frac{(n-h)!} {n!} \\
&= \binom{n} {h} \frac{(n-h)!} {n!} \\
&= \frac{1} {h!}
\end{align}$$
>
>The following steps are obvious
>
>However, I would like to show this by the pure theory from number series.
>
>We denote the probability for $n$ men as $P_n$, and it's easy to find that
>
>$$\begin{align}
P_n &= &\frac{n-1} {n} \frac{1} {n-1} P_{n-2} \\
&& +\frac{n-1} {n} \frac{n-2} {n-1} \frac{1} {n-2} P_{n-3}\\
&& \cdots \\
&& + \frac{n-1} {n} \cdots \frac{1} {2} P_{1}\\
&= \frac{1} {n} \sum_{i=1}^{n-2}P_i
\end{align}$$
>
>$$\begin{align}
nP_n &= P_1 + P_2 + \cdots + P_{n-2} \\
nP_n &= (n-1)P_{n-1} + P_{n-2} \\
n(P_n - P_{n-1}) &= -(P_{n-1} - P_{n-2})
\end{align}$$
>
>Almost there.

Probability and Statistics/Applied Random Process/Chap_01.ipynb


# INF-495, v0.01, Claudio Torres, ctorres@inf.utfsm.cl. DI-UTFSM
## Textbook: Computational Mathematical Modeling, An Integrated Approach Across Scales by Daniela Calvetti and Erkki Somersalo.

# Chapter 2

```python
import numpy as np
import scipy.sparse.linalg as sp
import sympy as sym
from scipy.linalg import toeplitz

import ipywidgets as widgets
from ipywidgets import IntSlider

import matplotlib.pyplot as plt
%matplotlib inline
from matplotlib import cm
from matplotlib.ticker import LinearLocator, FormatStrFormatter

plt.style.use('ggplot')

import matplotlib as mpl
mpl.rcParams['font.size'] = 14
mpl.rcParams['axes.labelsize'] = 20
mpl.rcParams['xtick.labelsize'] = 14
mpl.rcParams['ytick.labelsize'] = 14
sym.init_printing()

from scipy.integrate import odeint
from ipywidgets import interact
```

## 2.1 Simple compartment models

### Example 1:

```python
# Example 1, numerically.
def f_example_1(t,y,c,k1,k2):
    X = y[0]
    Y = y[1]

    Xp = c(t)-k1*X
    Yp = k1*X-k2*Y

    return np.array([Xp,Yp])

# Initial condition
y0 = np.array([500, 600])
a  = 1200
t1 = 12*60
t2 = 19*60
tau = 30
c  = lambda t: a*(np.exp(-((np.mod(t,24*60)-t1)**2)/(2*tau**2)) \
                +np.exp(-((np.mod(t,24*60)-t2)**2)/(2*tau**2))) \
                /(np.sqrt(2*np.pi*tau**2))

T_digest = 80
T_metabolic = 240
k1 = 1/T_digest
k2 = 1/T_metabolic

T = 2*24*60

# time where we want your solution
t = np.linspace(0, T, 100)
sol = odeint(f_example_1, y0, t, args=(c,k1,k2), tfirst=True)

plt.figure()
plt.plot(t, sol[:,0], 'b', label='X(t): Energy in gastric channel')
plt.plot(t, sol[:,1], 'r', label='Y(t): Energy in blood')
plt.legend(loc='right', bbox_to_anchor=(2, 0.5))
plt.xlabel('t')
plt.ylabel('kcal')
plt.grid(True)
plt.show()
```

```python
plt.figure()
T = 5*24*60
t = np.linspace(0, T, 1000)
plt.plot(t,c(t),'b')
plt.grid(True)
plt.plot()
```

## 2.1.1.- Simple population models

### Adding competition for resources: Logistic growth

```python
# Example 1, numerically.
def f_spm_logistic_growth(t,y,r,K):
    X = y
    Xp = r*(1-X/K)*X
    return Xp

# Initial condition
r = 1
K = 1000

T = 10

# time where we want your solution
t = np.linspace(0, T, 100)

plt.figure()
for y0 in np.linspace(0,2000,20):
    sol = odeint(f_spm_logistic_growth, y0, t, args=(r,K), tfirst=True)
    plt.plot(t, sol, 'b')
plt.xlabel('t')
plt.ylabel('population size')
plt.grid(True)
plt.show()
```

### Logistic growth with harvesting

```python
def f_logistic_growth_harvesting(t,y,r,K,h):
    X = y
    Xp = r*(1-X/K)*X-h
    return Xp

# Initial condition
r = 1
K = 1000
h = 100
print(K*r/4,h)

T = 8

Xplus = K/2*(1+np.sqrt(1-4*h/(K*r)))
Xminus = K/2*(1-np.sqrt(1-4*h/(K*r)))
print(Xplus,Xminus)

# time where we want your solution
t = np.linspace(0, T, 100)

plt.figure()
N=20
y0s = np.zeros(N+1)
y0s[0] = Xminus-0.5
y0s[1:] = np.linspace(Xminus,1200,N)
for y0 in y0s:
    sol = odeint(f_logistic_growth_harvesting, y0, t, args=(r,K,h), tfirst=True)
    plt.plot(t, sol, 'b')

plt.plot(t, sol*0+Xplus, 'r')
plt.plot(t, sol*0+Xminus, 'r')
plt.plot(t, sol*0, 'r--')
plt.xlabel('t')
plt.ylabel('population size')
plt.grid(True)
plt.show()
```

```python
plt.figure()
f = lambda X: -(r/K)*X**2+r*X-h
Xs = np.linspace(0,Xplus+200,100)
plt.plot(Xs, f(Xs), 'b')
plt.plot(Xs, f(Xs)*0, 'r')
plt.axvline(x=Xminus,color='k',label='$X_-$')
plt.axvline(x=Xplus,color='g',label='$X_+$')
plt.xlabel('X')
plt.legend(loc='right', bbox_to_anchor=(1.3, 0.5))
plt.grid(True)
plt.show()
```

### Models with delay

### Example 2

Differential equations with delay are not currently supported by Scipy, fortunately there are the following options:

- For Python: pydelay, http://pydelay.sourceforge.net
- For R: "PBSddesolve: Solver for delay differential equations", https://github.com/pbs-software/pbs-ddesolve
- For Python using PBSddesolve: https://github.com/hensing/PyDDE
- (*) For Python: https://github.com/Zulko/ddeint

```python
# See: https://github.com/Zulko/ddeint
from ddeint import ddeint

r = 2         # Growth rate
K = 100       # Carrying capacity
tau = 1       # Lag of crowding
X_hist = 50   # Constant history
T = 30        # Time simulation

def values_before_zero(t):
    return X_hist

def f_delay1(Y, t):
    return r*(1-Y(t - tau)/K)*Y(t)

t = np.linspace(0, T, 1000)
sol = ddeint(f_delay1, values_before_zero, t)

fig, ax = plt.subplots(1, figsize=(4, 4))
ax.plot(t, sol)
plt.show()
```

```python
r = 1.4       # Growth rate
K = 150       # Carrying capacity
tau1 = 1      # Lag of birth
tau2 = 5      # Lag of crowding
X_hist = 100  # Constant history
T = 300       # Time simulation
beta = 0.5    # Rate of death

def values_before_zero(t):
    return X_hist

def f_delay2(Y, t):
    return r*Y(t-tau1)-beta*Y(t)-(r/K)*Y(t - tau2)*Y(t)

t = np.linspace(0, T, 2000)
sol = ddeint(f_delay2, values_before_zero, t)

fig, ax = plt.subplots(1, figsize=(4, 4))
ax.plot(t, sol)
plt.show()
```

## 2.2.- Interacting population models

## 2.2.1.- Predator-prey models

### Example 3

```python
# Example 1, numerically.
def f_example_3(t,y,eta,gamma,eps):
    X = y[0]
    Y = y[1]

    Xp = eta*X*(1-Y/(eta/gamma))
    Yp = -eps*Y*(1-X/(eps/delta))

    return np.array([Xp,Yp])

# Initial condition

eta = 8-1/2
eps = 1/5
gamma = 1/100
delta = eps*gamma/(20*eta)

X_eq = eps/delta
Y_eq = eta/gamma

y0 = np.array([0.9*X_eq, 1.2*Y_eq])

T = 15

# time where we want your solution
t = np.linspace(0, T, 100)
sol = odeint(f_example_3, y0, t, args=(eta,gamma,eps), tfirst=True)

X_output = sol[:,0]
Y_output = sol[:,1]

fig, ax1 = plt.subplots()
color = 'tab:blue'
ax1.set_xlabel('time [years]')
ax1.set_ylabel('Prey population', color=color)
ax1.plot(t, X_output, color=color)
ax1.tick_params(axis='y', labelcolor=color)
ax2 = ax1.twinx()  # instantiate a second axes that shares the same x-axis
color = 'tab:green'
ax2.set_ylabel('Predator popolation', color=color)  # we already handled the x-label with ax1
ax2.plot(t, Y_output, color=color)
ax2.tick_params(axis='y', labelcolor=color)
fig.tight_layout()  # otherwise the right y-label is slightly clipped
plt.show()

plt.figure()
plt.plot(t,X_output,'b')
plt.plot(t,Y_output,'r')
plt.grid(True)
plt.show()
```

```python
plt.figure()
plt.plot(X_output,Y_output,'k')
plt.xlabel('prey')
plt.ylabel('predator')
plt.plot(X_eq,Y_eq,'b.',markersize=20)
plt.plot(y0[0],y0[1],'r.',markersize=20)
plt.text(X_eq,Y_eq,'  equilibrium state')
plt.text(y0[0],y0[1],'  initial state')
plt.show()
```

```python
fig = plt.figure()
ax = fig.gca(projection='3d')
ax.plot(X_output, Y_output, t,'b')
ax.view_init(elev=40,azim=230)
plt.xlabel('prey')
plt.ylabel('predator')
ax.set_zlabel('t')
plt.show()
```

## 2.2.2.- SIR models in epidemiology

```python
# Example 1, numerically.
def f_SIR(t,y,alpha,beta):
    S = y[0]
    I = y[1]
    R = y[2]

    Sp = -alpha*I*S
    Ip = alpha*I*S-beta*I
    Rp = beta*I

    return np.array([Sp,Ip,Rp])

# Initial condition
N = 1e6
alpha = 1e-6
beta = 1/3

S0 = 9e5
I0 = 1e5
y0 = np.array([S0, I0, N-S0-I0])

T = 20

# time where we want your solution
t = np.linspace(0, T, 100)
sol = odeint(f_SIR, y0, t, args=(alpha,beta), tfirst=True)

plt.figure()
plt.plot(t, sol[:,0], 'b', label='S(t)')
plt.plot(t, sol[:,1], 'r', label='I(t)')
plt.plot(t, sol[:,2], 'g', label='R(t)')
plt.legend(loc='right', bbox_to_anchor=(1.3, 0.5))
plt.xlabel('t')
plt.ylabel('number of individuals')
plt.grid(True)
plt.show()
```

## 2.2.3.- Endemic diseases

```python
# Example 1, numerically.
def f_SIR_endemic(t,y,alpha,beta,sigma,N):
    S = y[0]
    I = y[1]
    R = y[2]

    Sp = -alpha*I*S+sigma*N-sigma*S
    Ip = alpha*I*S-beta*I-sigma*I
    Rp = beta*I-sigma*R

    return np.array([Sp,Ip,Rp])

# Initial condition
N = 1e6
alpha = 1e-6
beta = 1/3
sigma = 1/50

S0 = 9e5
I0 = 1e5
y0 = np.array([S0, I0, N-S0-I0])

T = 150

# time where we want your solution
t = np.linspace(0, T, 100)
sol = odeint(f_SIR_endemic, y0, t, args=(alpha,beta,sigma,N), tfirst=True)

plt.figure()
plt.plot(t, sol[:,0], 'b', label='S(t)')
plt.plot(t, sol[:,1], 'r', label='I(t)')
plt.plot(t, sol[:,2], 'g', label='R(t)')
plt.legend(loc='right', bbox_to_anchor=(2, 0.5))
plt.xlabel('t')
plt.ylabel('number of individuals')
plt.grid(True)
plt.show()
```

# Project 1

https://github.com/projectmesa/mesa

https://towardsdatascience.com/introduction-to-mesa-agent-based-modeling-in-python-bcb0596e1c9a

# 2.2 Subcompartment models

 Example 5

```python
# Volumnes
Vol_ch = 50./1000
Vol_t  = 0.3/1000

# Initial conditions in chamber
Lac_ch_0 = 0.3
Glc_ch_0 = 1.6
O2_ch_0  = 0.6
CO2_ch_0 = 0.1
x0 = np.array([Lac_ch_0,Glc_ch_0,O2_ch_0,CO2_ch_0])

# Initial conditions in tissue
Lac_t_0  = 0.8
Glc_t_0  = 0.5
O2_t_0   = 0.14
CO2_t_0  = 0.2
ATP_t_0  = 6.2
ADP_t_0  = 0.8
y0 = np.array([Lac_t_0,Glc_t_0,O2_t_0,CO2_t_0,ATP_t_0,ADP_t_0])

# Maximum rates
T1 = 5e-5
T2 = 3e-5
V1 = 1e-4
V2 = 1e-4

# Affinities
M1 = 0.5
M2 = 0.5
K1 = 0.5
K2 = 0.22
mu1 = 0.1
mu2 = 0.1

#Fick's constants
lambda3 = 3e-3
lambda4 = 8e-3

# Constants defining the activity
t0  = 3. # min
tau = 10./60 # min

# Activity levels
a0      = 1e-5
delta_a = 1e-2

def f_example_5(t,z):
    x = z[:4]
    y = z[4:]

    Phi = np.zeros(4)
    Phi[0] = T1*((x[0]/(x[0]+M1))-(y[0]/(y[0]+M1)))
    Phi[1] = T2*((x[1]/(x[1]+M1))-(y[1]/(y[1]+M1)))
    Phi[2] = lambda3*(x[2]-y[2])
    Phi[3] = lambda4*(x[3]-y[3])

    p = y[5]/y[4]
    Psi1 = V1*(p/(p+mu1))*(y[1]/(y[1]+K1))
    rho1 = np.array([2.,-1,0,0,2,-2])*Psi1

    Psi2 = V2*(p/(p+mu2))*(y[1]*y[2]/(y[1]*y[2]+K2**2))
    rho2 = np.array([0.,-1,-6,6,32,-32])*Psi2

    a = a0+delta_a*(np.heaviside(t-(t0), 1)-np.heaviside(t-(t0+tau), 1))
    rho3 = np.array([0.,0,0,0,-1,1])*a

    dxdt = -Phi/Vol_ch

    dydt = np.zeros(6)
    dydt[:4] = Phi
    dydt += rho1+rho2+rho3
    dydt /=Vol_t
    return np.concatenate((dxdt,dydt), axis=0)

# Initial condition
z0 = np.concatenate((x0,y0), axis=0)

T = 10
# time where we want your solution
t = np.linspace(0, T, 100)

sol = odeint(f_example_5, z0, t, tfirst=True)

plt.figure(figsize=(10,10))

plt.subplot(221)
plt.plot(t, sol[:,4], 'b-', label='Lac_t(t)')
plt.plot(t, sol[:,5], 'b--', label='Glc_t(t)')
plt.legend(loc='best')
plt.axvline(x=t0,color='k',alpha=0.5)
plt.axvline(x=t0+tau,color='k',alpha=0.5)
plt.xlabel('t [min]')
plt.ylabel('concentration [mmol/l]')
plt.ylim([0.2,1])
plt.grid(True)

plt.subplot(222)
plt.plot(t, sol[:,6], 'b-', label='O2_t(t)')
plt.plot(t, sol[:,7], 'b--', label='CO2_t(t)')
plt.legend(loc='best')
plt.axvline(x=t0,color='k',alpha=0.5)
plt.axvline(x=t0+tau,color='k',alpha=0.5)
plt.xlabel('t [min]')
#plt.ylabel('concentration [mmol/l]')
plt.ylim([0,0.8])
plt.grid(True)

plt.subplot(223)
plt.plot(t, sol[:,8], 'b-', label='ATP_t(t)')
plt.plot(t, sol[:,9], 'b--', label='ADP_t(t)')
plt.legend(loc='best')
plt.axvline(x=t0,color='k',alpha=0.5)
plt.axvline(x=t0+tau,color='k',alpha=0.5)
plt.xlabel('t [min]')
plt.ylabel('concentration [mmol/l]')
plt.ylim([0,8])
plt.grid(True)

plt.subplot(224)
plt.plot(t, sol[:,2], 'b-', label='O2_Ch(t)')
plt.legend(loc='best')
plt.axvline(x=t0,color='k',alpha=0.5)
plt.axvline(x=t0+tau,color='k',alpha=0.5)
plt.xlabel('t [min]')
#plt.ylabel('concentration [mmol/l]')
plt.ylim([0.58,0.61])
plt.grid(True)

plt.tight_layout()
plt.show()
```

```python

```

SC4/Chapter 2 - Compartment models.ipynb