```python
import fractions
import math
import random

import numpy as np
import sympy
from typing import Callable, List, Optional, Sequence, Union

import cirq
```

```python
"""Function to compute the elements of Z_n."""
def multiplicative_group(n: int) -> List[int]:
    """Returns the multiplicative group modulo n.

    Args:
        n: Modulus of the multiplicative group.
    """
    assert n > 1
    group = [1]
    for x in range(2, n):
        if math.gcd(x, n) == 1:
            group.append(x)
    return group
```

```python
# multiplicative group modulo n = 15
"""Example of a multiplicative group."""
n = 15
print(f"The multiplicative group modulo n = {n} is:")
print(multiplicative_group(n))
```

    The multiplicative group modulo n = 15 is:
    [1, 2, 4, 7, 8, 11, 13, 14]

```python
# classical
"""Function for classically computing the order of an element of Z_n."""
def classical_order_finder(x: int, n: int) -> Optional[int]:
    """Computes smallest positive r such that x**r mod n == 1.

    Args:
        x: Integer whose order is to be computed, must be greater than one
           and belong to the multiplicative group of integers modulo n (which
           consists of positive integers relatively prime to n),
        n: Modulus of the multiplicative group.

    Returns:
        Smallest positive integer r such that x**r == 1 mod n.
        Always succeeds (and hence never returns None).

    Raises:
        ValueError when x is 1 or not an element of the multiplicative
        group of integers modulo n.
    """
    # Make sure x is both valid and in Z_n.
    if x < 2 or x >= n or math.gcd(x, n) > 1:
        raise ValueError(f"Invalid x={x} for modulus n={n}.")

    # Determine the order.
    r, y = 1, x
    while y != 1:
        y = (x * y) % n
        r += 1
    return r
```

```python
"""Example of (classically) computing the order of an element."""
n = 15  # The multiplicative group is [1, 2, 4, 7, 8, 11, 13, 14].
x = 8
r = classical_order_finder(x, n)

# Check that the order is indeed correct.
print(f"x^r mod n = {x}^{r} mod {n} = {x**r % n}")
```

    x^r mod n = 8^4 mod 15 = 1

```python
"""Example of defining an arithmetic (quantum) operation in Cirq."""
class Adder(cirq.ArithmeticOperation):
    """Quantum addition."""
    def __init__(self, target_register, input_register):
        self.input_register = input_register
        self.target_register = target_register

    def registers(self):
        return self.target_register, self.input_register

    def with_registers(self, *new_registers):
        return Adder(*new_registers)

    def apply(self, target_value, input_value):
        return target_value + input_value
```

```python
"""Example of using an Adder in a circuit."""
# Two qubit registers.
qreg1 = cirq.LineQubit.range(2)
qreg2 = cirq.LineQubit.range(2, 4)

# Define the circuit.
circ = cirq.Circuit(
    cirq.ops.X.on(qreg1[0]),
    cirq.ops.X.on(qreg2[1]),
    Adder(input_register=qreg1, target_register=qreg2),
    cirq.measure_each(*qreg1),
    cirq.measure_each(*qreg2)
)

# Display it.
print("Circuit:\n")
print(circ)

# Print the measurement outcomes.
print("\n\nMeasurement outcomes:\n")
print(cirq.sample(circ, repetitions=5).data)
```

    Circuit:

    0: ───X───#3──────────────────────────────────────────M───
              │
    1: ───────#4──────────────────────────────────────────M───
              │
    2: ───────<__main__.Adder object at 0x7f7859d223a0>───M───
              │
    3: ───X───#2──────────────────────────────────────────M───

    Measurement outcomes:

       0  1  2  3
    0  1  0  1  1
    1  1  0  1  1
    2  1  0  1  1
    3  1  0  1  1
    4  1  0  1  1

```python
"""Example of the unitary of an Adder operation."""
cirq.unitary(
    Adder(target_register=cirq.LineQubit.range(2),
          input_register=1)
).real
```

    array([[0., 0., 0., 1.],
           [1., 0., 0., 0.],
           [0., 1., 0., 0.],
           [0., 0., 1., 0.]])

```python
"""Defines the modular exponential operation used in Shor's algorithm."""
class ModularExp(cirq.ArithmeticOperation):
    """Quantum modular exponentiation.

    This class represents the unitary which multiplies base raised to exponent
    into the target modulo the given modulus. More precisely, it represents the
    unitary V which computes modular exponentiation x**e mod n:

        V|y⟩|e⟩ = |y * x**e mod n⟩ |e⟩     0 <= y < n
        V|y⟩|e⟩ = |y⟩ |e⟩                  n <= y

    where y is the target register, e is the exponent register, x is the base
    and n is the modulus. Consequently,

        V|y⟩|e⟩ = (U**e|y)|e⟩

    where U is the unitary defined as

        U|y⟩ = |y * x mod n⟩      0 <= y < n
        U|y⟩ = |y⟩                n <= y
    """
    def __init__(
        self,
        target: Sequence[cirq.Qid],
        exponent: Union[int, Sequence[cirq.Qid]],
        base: int,
        modulus: int
    ) -> None:
        if len(target) < modulus.bit_length():
            raise ValueError(f'Register with {len(target)} qubits is too small '
                             f'for modulus {modulus}')
        self.target = target
        self.exponent = exponent
        self.base = base
        self.modulus = modulus

    def registers(self) -> Sequence[Union[int, Sequence[cirq.Qid]]]:
        return self.target, self.exponent, self.base, self.modulus

    def with_registers(
            self,
            *new_registers: Union[int, Sequence['cirq.Qid']],
    ) -> cirq.ArithmeticOperation:
        if len(new_registers) != 4:
            raise ValueError(f'Expected 4 registers (target, exponent, base, '
                             f'modulus), but got {len(new_registers)}')
        target, exponent, base, modulus = new_registers
        if not isinstance(target, Sequence):
            raise ValueError(
                f'Target must be a qubit register, got {type(target)}')
        if not isinstance(base, int):
            raise ValueError(
                f'Base must be a classical constant, got {type(base)}')
        if not isinstance(modulus, int):
            raise ValueError(
                f'Modulus must be a classical constant, got {type(modulus)}')
        return ModularExp(target, exponent, base, modulus)

    def apply(self, *register_values: int) -> int:
        assert len(register_values) == 4
        target, exponent, base, modulus = register_values
        if target >= modulus:
            return target
        return (target * base**exponent) % modulus

    def _circuit_diagram_info_(
            self,
            args: cirq.CircuitDiagramInfoArgs,
    ) -> cirq.CircuitDiagramInfo:
        assert args.known_qubits is not None
        wire_symbols: List[str] = []
        t, e = 0, 0
        for qubit in args.known_qubits:
            if qubit in self.target:
                if t == 0:
                    if isinstance(self.exponent, Sequence):
                        e_str = 'e'
                    else:
                        e_str = str(self.exponent)
                    wire_symbols.append(
                        f'ModularExp(t*{self.base}**{e_str} % {self.modulus})')
                else:
                    wire_symbols.append('t' + str(t))
                t += 1
            if isinstance(self.exponent, Sequence) and qubit in self.exponent:
                wire_symbols.append('e' + str(e))
                e += 1
        return cirq.CircuitDiagramInfo(wire_symbols=tuple(wire_symbols))
```

```python
"""Create the target and exponent registers for phase estimation,
and see the number of qubits needed for Shor's algorithm.
"""
n = 15
L = n.bit_length()

# The target register has L qubits.
target = cirq.LineQubit.range(L)

# The exponent register has 2L + 3 qubits.
exponent = cirq.LineQubit.range(L, 3 * L + 3)

# Display the total number of qubits to factor this n.
print(f"To factor n = {n} which has L = {L} bits, we need 3L + 3 = {3 * L + 3} qubits.")
```

    To factor n = 15 which has L = 4 bits, we need 3L + 3 = 15 qubits.

```python
"""See (part of) the unitary for a modular exponential operation."""
# Pick some element of the multiplicative group modulo n.
x = 5

# Display (part of) the unitary. Uncomment if n is small enough.
# cirq.unitary(ModularExp(target, exponent, x, n))
```

```python
"""Function to make the quantum circuit for order finding."""
def make_order_finding_circuit(x: int, n: int) -> cirq.Circuit:
    """Returns quantum circuit which computes the order of x modulo n.

    The circuit uses Quantum Phase Estimation to compute an eigenvalue of
    the unitary

        U|y⟩ = |y * x mod n⟩      0 <= y < n
        U|y⟩ = |y⟩                n <= y

    Args:
        x: positive integer whose order modulo n is to be found
        n: modulus relative to which the order of x is to be found

    Returns:
        Quantum circuit for finding the order of x modulo n
    """
    L = n.bit_length()
    target = cirq.LineQubit.range(L)
    exponent = cirq.LineQubit.range(L, 3 * L + 3)
    return cirq.Circuit(
        cirq.X(target[L - 1]),
        cirq.H.on_each(*exponent),
        ModularExp(target, exponent, x, n),
        cirq.qft(*exponent, inverse=True),
        cirq.measure(*exponent, key='exponent'),
    )
```

```python
"""Example of the quantum circuit for period finding."""
n = 15
x = 7
circuit = make_order_finding_circuit(x, n)
print(circuit)
```

    0: ────────ModularExp(t*7**e % 15)────────────────────────────
               │
    1: ────────t1─────────────────────────────────────────────────
               │
    2: ────────t2─────────────────────────────────────────────────
               │
    3: ────X───t3─────────────────────────────────────────────────
               │
    4: ────H───e0────────────────────────qft^-1───M('exponent')───
               │                         │        │
    5: ────H───e1────────────────────────#2───────M───────────────
               │                         │        │
    6: ────H───e2────────────────────────#3───────M───────────────
               │                         │        │
    7: ────H───e3────────────────────────#4───────M───────────────
               │                         │        │
    8: ────H───e4────────────────────────#5───────M───────────────
               │                         │        │
    9: ────H───e5────────────────────────#6───────M───────────────
               │                         │        │
    10: ───H───e6────────────────────────#7───────M───────────────
               │                         │        │
    11: ───H───e7────────────────────────#8───────M───────────────
               │                         │        │
    12: ───H───e8────────────────────────#9───────M───────────────
               │                         │        │
    13: ───H───e9────────────────────────#10──────M───────────────
               │                         │        │
    14: ───H───e10───────────────────────#11──────M───────────────

```python
# example of measurement outcomes
```

```python
"""Measuring Shor's period finding circuit."""
circuit = make_order_finding_circuit(x=5, n=6)
res = cirq.sample(circuit, repetitions=8)

print("Raw measurements:")
print(res)

print("\nInteger in exponent register:")
print(res.data)
```

    Raw measurements:
    exponent=01110111, 00000000, 00000000, 00000000, 00000000, 00000000, 00000000, 00000000, 00000000

    Integer in exponent register:
       exponent
    0         0
    1       256
    2       256
    3       256
    4         0
    5       256
    6       256
    7       256

```python
# classical post processing
def process_measurement(result: cirq.Result, x: int, n: int) -> Optional[int]:
    """Interprets the output of the order finding circuit.

    Specifically, it determines s/r such that exp(2πis/r) is an eigenvalue
    of the unitary

        U|y⟩ = |xy mod n⟩  0 <= y < n
        U|y⟩ = |y⟩         n <= y

    then computes r (by continued fractions) if possible, and returns it.

    Args:
        result: result obtained by sampling the output of the
            circuit built by make_order_finding_circuit

    Returns:
        r, the order of x modulo n or None.
    """
    # Read the output integer of the exponent register.
    exponent_as_integer = result.data["exponent"][0]
    exponent_num_bits = result.measurements["exponent"].shape[1]
    eigenphase = float(exponent_as_integer / 2**exponent_num_bits)

    # Run the continued fractions algorithm to determine f = s / r.
    f = fractions.Fraction.from_float(eigenphase).limit_denominator(n)

    # If the numerator is zero, the order finder failed.
    if f.numerator == 0:
        return None

    # Else, return the denominator if it is valid.
    r = f.denominator
    if x**r % n != 1:
        return None
    return r
```

```python
"""Example of the classical post-processing."""
# Set n and x here
n = 6
x = 5

print(f"Finding the order of x = {x} modulo n = {n}\n")
measurement = cirq.sample(circuit, repetitions=1)
print("Raw measurements:")
print(measurement)

print("\nInteger in exponent register:")
print(measurement.data)

r = process_measurement(measurement, x, n)
print("\nOrder r =", r)
if r is not None:
    print(f"x^r mod n = {x}^{r} mod {n} = {x**r % n}")
```

    Finding the order of x = 5 modulo n = 6

    Raw measurements:
    exponent=1, 0, 0, 0, 0, 0, 0, 0, 0

    Integer in exponent register:
       exponent
    0       256

    Order r = 2
    x^r mod n = 5^2 mod 6 = 1

```python
# quantum order finder
def quantum_order_finder(x: int, n: int) -> Optional[int]:
    """Computes smallest positive r such that x**r mod n == 1.

    Args:
        x: integer whose order is to be computed, must be greater than one
           and belong to the multiplicative group of integers modulo n (which
           consists of positive integers relatively prime to n),
        n: modulus of the multiplicative group.
    """
    # Check that the integer x is a valid element of the multiplicative group
    # modulo n.
    if x < 2 or n <= x or math.gcd(x, n) > 1:
        raise ValueError(f'Invalid x={x} for modulus n={n}.')

    # Create the order finding circuit.
    circuit = make_order_finding_circuit(x, n)

    # Sample from the order finding circuit.
    measurement = cirq.sample(circuit)

    # Return the processed measurement result.
    return process_measurement(measurement, x, n)
```

```python
"""Functions for factoring from start to finish."""
def find_factor_of_prime_power(n: int) -> Optional[int]:
    """Returns non-trivial factor of n if n is a prime power, else None."""
    for k in range(2, math.floor(math.log2(n)) + 1):
        c = math.pow(n, 1 / k)
        c1 = math.floor(c)
        if c1**k == n:
            return c1
        c2 = math.ceil(c)
        if c2**k == n:
            return c2
    return None

def find_factor(
    n: int,
    order_finder: Callable[[int, int], Optional[int]] = quantum_order_finder,
    max_attempts: int = 30
) -> Optional[int]:
    """Returns a non-trivial factor of composite integer n.

    Args:
        n: Integer to factor.
        order_finder: Function for finding the order of elements of the
            multiplicative group of integers modulo n.
        max_attempts: number of random x's to try, also an upper limit
            on the number of order_finder invocations.

    Returns:
        Non-trivial factor of n or None if no such factor was found.
        Factor k of n is trivial if it is 1 or n.
    """
    # If the number is prime, there are no non-trivial factors.
    if sympy.isprime(n):
        print("n is prime!")
        return None

    # If the number is even, two is a non-trivial factor.
    if n % 2 == 0:
        return 2

    # If n is a prime power, we can find a non-trivial factor efficiently.
    c = find_factor_of_prime_power(n)
    if c is not None:
        return c

    for _ in range(max_attempts):
        # Choose a random number between 2 and n - 1.
        x = random.randint(2, n - 1)

        # Most likely x and n will be relatively prime.
        c = math.gcd(x, n)

        # If x and n are not relatively prime, we got lucky and found
        # a non-trivial factor.
        if 1 < c < n:
            return c

        # Compute the order r of x modulo n using the order finder.
        r = order_finder(x, n)

        # If the order finder failed, try again.
        if r is None:
            continue

        # If the order r is even, try again.
        if r % 2 != 0:
            continue

        # Compute the non-trivial factor.
        y = x**(r // 2) % n
        assert 1 < y < n
        c = math.gcd(y - 1, n)
        if 1 < c < n:
            return c

    print(f"Failed to find a non-trivial factor in {max_attempts} attempts.")
    return None
```

```python
"""Example of factoring via Shor's algorithm (order finding)."""
# Number to factor
n = 184573

# Attempt to find a factor
p = find_factor(n, order_finder=classical_order_finder)
q = n // p

print("Factoring n = pq =", n)
print("p =", p)
print("q =", q)
```

    Factoring n = pq = 184573
    p = 487
    q = 379

```python
"""Check the answer is correct."""
p * q == n
```

    True

```python

```


# Juliaによる数値計算

## 数理最適化

- **数理最適化**:
    - 与えられた制約の中で、ある関数の値を最大化（あるいは最小化）すること
    - 現実の問題のポイントを整理して数式で表し、数式にあった解法（アルゴリズム）で最適な解を求める
    - 数理最適化は、意思決定にかかわる科学的アプローチであるオペレーションズリサーチの一分野として発展してきたが、今日においては社会にとって欠かせない技術の一つである

ここでは、機械学習アルゴリズムでよく使われる数理最適化手法を見ていく

### 線形計画問題
線形計画問題そのものは機械学習アルゴリズムの中で現れることはあまりないが、数理最適化問題の基礎となる問題であるため、ここで軽く触れておく

次のような問題を考える

---

ある工場では製品 X, Yを製造している

これらは原料 A, B, C から作られており、それぞれの製品1個（1単位）を作るのに必要な原料（kg）は次の表である

$$
\begin{array}{cccc}
& A & B & C \\ \hline
X & 1 & 2 & 2 \\ \hline
Y & 4 & 3 & 1
\end{array}
$$

工場の倉庫にある原料の量（kg）は次の表のとおりである

$$
\begin{array}{ccc}
A & B & C \\ \hline
1700 & 1400 & 1000
\end{array}
$$

製品 X, Y を売ると、それぞれ利益は1つあたり3ドルと4ドルになる

このとき、利益を最大化するには X, Y をそれぞれどれだけ製造すれば良いか

---

ここでは話を簡単にするため、製造する量は整数とは限らないとする

これを式で表してみる

まず製品 X と Y を製造する量をそれぞれ $x$, $y$ とおくと、このとき使われる原料 A は $x+4y$ となるため、原料 A の在庫についての制約から、

$$
x + 4y \leq 1700
$$

となる

同様に原料 B, C について考えると、以下の式を得ることができる

$$
\begin{align}
2x + 3y &\leq 1400 \\
2x + y &\leq 1000
\end{align}
$$

また、各製品の製造量は 0 以上である必要があるため、

$$
x \geq 0,\ y \geq 0
$$

となる

これらの制約条件のもと、利益 $3x+4y$ を最大化するという問題である

以上のことをまとめて次のように書くことがある

$$
\begin{align}
\rm{Maximize} \quad & 3x + 4y \\
\rm{Subject\ to} \quad & x + 4y \leq 1700 \\
& 2x + 3y \leq 1400 \\
& 2x + y \leq 1000 \\
& x \geq 0 \\
& y \geq 0
\end{align}
$$

上記のように最適化したい関数を先に記述し、最大化したいのか最小化したいのかを $\rm{Maximaize}$ または $\rm{Minimize}$ で表す

ここに書かれる関数を **目的関数** と呼ぶ

目的関数の後ろに $\rm{Subject\ to}$ で制約式を書くことになる

ここでは $x$ と $y$ の値を変化させるが、これらは **変数** と呼ばれ、制約式を満たすような変数の値を **実行可能解** と呼ぶ

また、目的関数を最適化（最大化もしくは最小化）する変数の値を **最適解** と呼び、その時の目的関数の値を **最適値** と呼ぶ

変数を明示したい場合は、目的関数を $\rm{Maximize}_{x, y}\ 3x+4y$ のように記述する

今回のように、目的関数も制約式も1次式（つまり線形）である最適化問題を特に **線形計画問題** と呼ぶ

制約式を満たす領域を図示すると以下のようになる

上図の赤色部分の点は境界線上の点も含めて、全て実行可能解ということになる

最適解を求めるには、この赤色領域について目的関数 $3x+4y$ が最大となる点を見つければ良い

ここでは任意定数 $k$ について $3x+4y=k$ を満たす点の集合を考えてみる

$3x+4y=k$ を満たす点の集合は直線になるが、$(x,y)$ が赤色領域内にあるとき $3x+4y$ を最大化する問題は、つまり直線 $3x+4y=k$ が赤色部分を通過するという条件で $k$ を最大化するという問題と言える

$k$ を変えていったとき、$3x+4y=k$ で表される直線群は全て平行になるため、これは $3x+4y$ の等高線であると見なすことが出来る

従って、直線を平行にずらしていきながら、ぎりぎり赤色部分と共有点をもつ場合を考えれば良いことになる

今回の場合は、図より $3x+4y=k$ が点 $(400,200)$ を通過するときが最適解であり、これを $x$, $y$ に代入して $3\times400+2\times400=2000$ が最適値となる

#### 線形計画法の一般型
ここまでは例として2変数の場合の線形計画問題を見てきたが、一般には$n$変数の場合の線形計画問題を考えることが出来る

$n$次元ベクトル $\boldsymbol x$ を変数とすると、線形計画法の一般型は次のように表される

```julia

```


# Kalman.jl

The [Kalman.jl](https://github.com/wkearn/Kalman.jl) package aims to provide a Julian interface for various kinds of Kalman filter.

```
using Kalman, Winston
```

The Kalman filter is a way to estimate the state of a system given some knowledge of the (possibly stochastic) dynamics of that system and some noisy data collected from the system. Basically you take your state estimate, run it through the system model to advance it in time (along with the covariance matrix of your estimate), then you update your state estimate and state covariance matrix with the information gained from the data you have collected.

We'll try out the Kalman filter using an example from [Welch and Bishop, 2006](http://www.cs.unc.edu/~welch/media/pdf/kalman_intro.pdf).

Imagine we have a voltmeter taking measurements of a noisy but, on average, constant circuit. We first need a linear state-space model of this system that looks like $ x_{k+1} = Ax_k+w_k$. Because the voltage of the circuit doesn't change from time to time, we set $A=1$ and find our model $ x_{k+1} = x_k + w_k$ where $w_k$ is a vector of white Gaussian noise with variance $Q$.

The second half of the Kalman filter model is an observation model which relates the measurements you take from the system to the system state. It takes the form $ z_{k} = Hx_k + v_k$ where $v_k$ is white Gaussian noise with variance $R$. For our example, we just take direct measurements of the voltage, so $H=1$.

## Types

Now we look at how we can represent those models in Julia.

```
type LinearModel <: Model
    a::Matrix
    g::Matrix
    q::Matrix
end
```
A `LinearModel` takes a matrix `a` representing the state-transition matrix, `g` which explains how the process noise affects the system state, and `q` which is the process noise covariance matrix.

We can set one of those up for our model now. Our two transition matrices are just 1 as above. We'll give our process some small (but non-zero!) noise. The circuit isn't going to be perfectly constant.

```
f = LinearModel([1]',[1]',[1e-10]')
```

    LinearModel(1x1 Array{Int64,2}:
     1,1x1 Array{Int64,2}:
     1,1x1 Array{Float64,2}:
     1.0e-10)

The observation model is represented with a `LinearObservationModel` type

```
type LinearObservationModel <: ObservationModel
    h::Matrix
    r::Matrix
end
```

with definitions as above. For our example h is just `[1]'`, and we'll give our voltmeter $0.1\ \mathrm{V}$ RMS noise.

```
z = LinearObservationModel([1]',[0.01]')
```

    LinearObservationModel(1x1 Array{Int64,2}:
     1,1x1 Array{Float64,2}:
     0.01)

Finally, we need a way to represent the state of our system. To completely describe the state, we need both a state vector and a state covariance matrix

```
type State
    x::Vector
    p::Matrix
end
```

We have to seed our Kalman filter with an initial state and covariance estimate. It turns out that the covariance will converge regardless of the starting value. For our example as well, the state will converge, but there are systems with dynamics which might lead the state into a local minimum, so it's important to make a reasonable estimate for the state.

We'll start with $\hat{x}_0 = 0$ and $P_0 = 0.5$

```
x0 = State([0.0],[1.0]')
```

    State{Float64}([0.0],1x1 Array{Float64,2}:
     1.0)

Now we have everything to make our Kalman Filter. We just stick the two models and the state into a `BasicKalmanFilter` type

```
type BasicKalmanFilter <: LinearKalmanFilter
    x::State
    f::LinearModel
    z::LinearObservationModel
    adv::Bool
end
```

The `adv` field is a bad way of ensuring that we first advance our filter in time and then update it with some measurements. That may go away.

```
kf = BasicKalmanFilter(x0,f,z,false)
```

    BasicKalmanFilter(State{Float64}([0.0],1x1 Array{Float64,2}:
     1.0),LinearModel(1x1 Array{Int64,2}:
     1,1x1 Array{Int64,2}:
     1,1x1 Array{Float64,2}:
     1.0e-10),LinearObservationModel(1x1 Array{Int64,2}:
     1,1x1 Array{Float64,2}:
     0.01),false)

Now we need some data. Data to be fed to the Kalman filter is wrapped in an `Observation` type (which should probably just be a typealias for a Vector). Let's assume that our true voltage $x = -0.37727\ \mathrm{V}$.

```
x = -0.37727
y = map(y->Observation([y]),x+0.1*randn(50))
```

    50-element Array{Observation{Float64},1}:
     Observation{Float64}([-0.556689])
     Observation{Float64}([-0.28849])
     Observation{Float64}([-0.429203])
     Observation{Float64}([-0.27673])
     Observation{Float64}([-0.461203])
     Observation{Float64}([-0.348198])
     Observation{Float64}([-0.587995])
     Observation{Float64}([-0.333402])
     Observation{Float64}([-0.413611])
     Observation{Float64}([-0.502472])
     Observation{Float64}([-0.43491])
     Observation{Float64}([-0.465556])
     Observation{Float64}([-0.275459])
     ⋮
     Observation{Float64}([-0.4012])
     Observation{Float64}([-0.494106])
     Observation{Float64}([-0.516698])
     Observation{Float64}([-0.389034])
     Observation{Float64}([-0.283263])
     Observation{Float64}([-0.198191])
     Observation{Float64}([-0.318942])
     Observation{Float64}([-0.229493])
     Observation{Float64}([-0.260345])
     Observation{Float64}([-0.485391])
     Observation{Float64}([-0.297279])
     Observation{Float64}([-0.203137])

Now we can run our filter by sequentially calling `predict(kf::BasicKalmanFilter)` and `update(kf::BasicKalmanFilter,y::Observation)` or by simply calling `predictupdate(kf,y)`. We'll also store the state and variance information and plot it up with Winston.

```
xs = x0.x[1]*ones(50)
ps = x0.p[1]*ones(50)

for i = 1:49
    predictupdate!(kf,y[i])
    xs[i+1] = kf.x.x[1]
    ps[i+1] = kf.x.p[1]
end
```

```
plot([0,50],[x,x],"b")
oplot(map(y->y.y[1],y),"r.")
oplot(xs,"g")
xlim(0,50)
```

```
plot(ps[2:50],"g")
```

## Unscented Kalman filters.

Unscented Kalman filters let you apply the prediction-update method of the Kalman filter to nonlinear models. They do this by deterministically sampling a set of "sigma points" which are then run through the filter equations to reconstruct the state and covariance matrices after a nonlinear transformation.

Unscented Kalman filters aren't pushed to Github yet, and there are some restrictions on the kind of model you can use for it (additive Gaussian noise for now; you might be able to get away with non-Gaussian noise, but multiplicative noise definitely won't work).

Still, let's see if we can't make them work. We'll take the example from Kandepu et al. (2008), a van der Pol Oscillator.

The equations of motion are

\begin{align}
\dot{x}_1 &= -x_2 \\
\dot{x}_2 &= -\mu (1-x_1^2)x^2 + x_1
\end{align}

where $\mu=0.2$.

```
include(Pkg.dir("Kalman","sandbox","unscented.jl"))
```

    predictupdate! (generic function with 3 methods)

```
const μ = 0.2

x0 = State([0.,5],5.0*eye(2))
Q = 1e-3*eye(2)
R = [0.1 0;
    0 1e-3]

function f2(x::Vector,dt::Float64)
    x1 = zeros(x)
    x1[1] = x[1] + dt*-x[2]
    x1[2] = x[2] + dt*(-μ*(1-x[1]^2)*x[2]+x[1])
    x1
end

h(x) = x

dt = 0.01

xs = fill(zeros(x0.x),4000)
xs[1] = [1.4,0]
x = [1.4,0]
for i = 2:4000
    x = f2(x,dt)
    xs[i] = x
end

ys = xs .+ map(y->sqrt(R)*randn(2),1:4000)
```

    4000-element Array{Array{Float64,1},1}:
     [1.09289,-0.0238201]
     [1.46761,0.0453131]
     [1.35845,0.0130256]
     [1.10434,0.0656155]
     [1.18755,0.0145539]
     [1.32602,0.0972942]
     [1.25023,0.136334]
     [0.902791,0.112868]
     [1.22303,0.0854962]
     [1.11071,0.148996]
     [1.36694,0.151589]
     [1.71204,0.152962]
     [1.61606,0.172073]
     ⋮
     [0.0785064,0.015488]
     [-0.771996,0.0356125]
     [0.339114,0.0404495]
     [0.108062,0.0174942]
     [0.323293,0.00062621]
     [-0.0526205,0.041678]
     [-0.0474085,0.00729233]
     [0.202621,0.0664006]
     [-0.329399,0.0489221]
     [-0.157211,0.0354753]
     [0.131959,0.0343319]
     [0.109813,0.0808878]

```
fm = AdditiveUnscentedModel(x->f2(x,dt),Q)
zm = AdditiveUnscentedObservationModel(h,R)
p0 = 5.0*eye(2)
kf = AdditiveUnscentedKalmanFilter(State([0,5.0],p0),fm,zm,0.1,2.0,0.0)

xs1 = 0.0*ones(4000)
xs2 = 5.0*ones(4000)
ps = 5.0*ones(4000)

for i = 2:4000
    predictupdate!(kf,Observation([ys[i]]))
    xs1[i] = kf.x.x[1]
    xs2[i] = kf.x.x[2]
    ps[i] = kf.x.p[1]
end
```

```
plot(xs1,"k")
```


# Exercise 10

```python
%matplotlib notebook
import matplotlib.pyplot as plt
import numpy as np
from math import *
from scipy.signal import *
```

```python
b = [ 0.85, 0.1]
a = [ 1, -0.4, -0.05 ]
r, p, k = residue(b, a)
print("residues:", r)
print("poles:", p)
print("direct polynomial term:", k)
```

    residues: [-0.025  0.875]
    poles: [-0.1  0.5]
    direct polynomial term: [0.]

$$ H(s) = \frac{0.85 s + 0.1}{s^2 - 0.4 s - 0.05} = - \frac{0.025}{s + 0.1} + \frac{0.875}{s - 0.5} $$

$$ h(t) = -0.025 \exp(-0.1 t) + 0.875 \exp(0.5 t) $$

---
# Exercise 11

```python
b = [ 2, -11, 12]
a = [ 1, -7, 18, -20, 8 ]
r, p, k = residue(b, a)
print("residues:", r)
print("poles:", p)
print("direct polynomial term:", k)
```

    residues: [-3.+0.j  3.+0.j -1.+0.j -2.+0.j]
    poles: [1.+0.j 2.+0.j 2.+0.j 2.+0.j]
    direct polynomial term: [0.]

$$ H(s) = \frac{2 s^2 - 11 s + 12}{s^4 - 7 s^3 + 18 s^2 - 20 s + 8} =
          -\frac{3}{s-1} +\frac{3}{s-2}-\frac{1}{(s-2)^2}-\frac{2}{(s-2)^3} $$
$$ h(t) = -3e^t + 3e^{2t} -t e^{2t} -t^2 e^{2t} $$

```python
import sympy as sp
sp.init_printing()
s, t = sp.symbols('s t')
Hs = (2*s**2 -11*s +12) / (s**4-7*s**3+18*s**2-20*s+8)
sp.Eq(Hs, Hs.apart())
```

```python
sp.inverse_laplace_transform(Hs.apart(), s, t)
```

---
# Exercise 12

```python
Hs = 2*s/(s+1)/(s+3)
sp.Eq(Hs, Hs.apart())
```

```python
sp.inverse_laplace_transform(2*s/(s+1)/(s+3), s, t)
```

---
# Exercise 13

```python
f0 = 100 # Hertz
b = [ 2, -11, 12 ]
a = [ 1, -7, 18, -20, 8 ]

filter = lti(b, a)
filter
```

    TransferFunctionContinuous(
    array([  2., -11.,  12.]),
    array([  1.,  -7.,  18., -20.,   8.]),
    dt: None
    )

```python
Omega, Gain, Phase = filter.bode()
Omega *= f0

fig, (ax1, ax2) = plt.subplots(2,1, sharex=True)
ax1.semilogx(Omega, Gain)
ax1.set_ylabel('gain [dB]')
ax1.grid()
ax2.semilogx(Omega, Phase)
ax2.set_xlabel('frequency [Hz]')
ax2.set_ylabel('phase [deg]')
ax2.grid()
```

    <IPython.core.display.Javascript object>

---


# Univariate statistics

Basics univariate statistics are required to explore dataset:

- Discover associations between a variable of interest and potential predictors. It is strongly recommended to start with simple univariate methods before moving to complex multivariate predictors.

- Assess the prediction performances of machine learning predictors.

- Most of the univariate statistics are based on the linear model which is one of the main model in machine learning.

## Libraries

**Data**

```python
import numpy as np
import pandas as pd
```

**Plots**

```python
import matplotlib.pyplot as plt
import seaborn as sns
```

**Statistics**

- Basic: [scipy.stats](https://docs.scipy.org/doc/scipy/reference/tutorial/stats.html)
- Advanced: [statsmodels](https://www.statsmodels.org/). [statsmodels API](https://www.statsmodels.org/stable/api.html):
    * `statsmodels.api`: Cross-sectional models and methods. Canonically imported using `import statsmodels.api as sm`.
    * `statsmodels.formula.api`: A convenience interface for specifying models using formula strings and DataFrames. Canonically imported using import `statsmodels.formula.api as smf`
    * `statsmodels.tsa.api`: Time-series models and methods. Canonically imported using `import statsmodels.tsa.api as tsa`.

```python
import scipy.stats
import statsmodels.api as sm
#import statsmodels.stats.api as sms
import statsmodels.formula.api as smf
from statsmodels.stats.stattools import jarque_bera
```

```python
%matplotlib inline
```

**Datasets**

Salary

```python
try:
    salary = pd.read_csv("../datasets/salary_table.csv")
except:
    url = 'https://github.com/duchesnay/pystatsml/raw/master/datasets/salary_table.csv'
    salary = pd.read_csv(url)
```

Iris

```python
# Load iris datset
iris = sm.datasets.get_rdataset("iris").data
iris.columns = [s.replace('.', '') for s in iris.columns]
```

## Estimators of the main statistical measures

### Mean

Properties of the expected value operator $\operatorname{E}(\cdot)$ of a random variable $X$

\begin{align}
    E(X + c) &= E(X) + c \\
    E(X + Y) &= E(X) + E(Y) \\
    E(aX)    &= a E(X)
\end{align}

The estimator $\bar{x}$ on a sample of size $n$: $x = x_1, ..., x_n$ is given by
$$
    \bar{x} = \frac{1}{n} \sum_i x_i
$$

$\bar{x}$ is itself a random variable with properties:

- $E(\bar{x}) = \bar{x}$,

- $\operatorname{Var}(\bar{x}) = \frac{\operatorname{Var}(X)}{n}$.

### Variance

$$
    \operatorname{Var}(X) = E((X - E(X))^2) =  E(X^2) - (E(X))^2
$$

The estimator is
$$
    \sigma_x^2 = \frac{1}{n-1} \sum_i (x_i - \bar{x})^2
$$

Note here the subtracted 1 degree of freedom (df) in the divisor. In standard statistical practice, $df=1$ provides an unbiased estimator of the variance of a hypothetical infinite population. With $df=0$ it instead provides a maximum likelihood estimate of the variance for normally distributed variables.

### Standard deviation

$$
    \operatorname{Std}(X) = \sqrt{\operatorname{Var}(X)}
$$

The estimator is simply $\sigma_x = \sqrt{\sigma_x^2}$.

### Covariance

$$
    \operatorname{Cov}(X, Y) = E((X - E(X))(Y - E(Y))) =  E(XY) - E(X)E(Y).
$$

Properties:

\begin{align}
    \operatorname{Cov}(X, X) &= \operatorname{Var}(X)\\
    \operatorname{Cov}(X, Y) &= \operatorname{Cov}(Y, X)\\
    \operatorname{Cov}(cX, Y) &= c \operatorname{Cov}(X, Y)\\
    \operatorname{Cov}(X+c, Y) &= \operatorname{Cov}(X, Y)\\
\end{align}

The estimator with $df=1$ is
$$
    \sigma_{xy} = \frac{1}{n-1} \sum_i (x_i - \bar{x}) (y_i - \bar{y}).
$$

### Correlation

$$
    \operatorname{Cor}(X, Y) = \frac{\operatorname{Cov}(X, Y)}{\operatorname{Std}(X)\operatorname{Std}(Y)}
$$

The estimator is
$$
    \rho_{xy} = \frac{\sigma_{xy}}{\sigma_{x} \sigma_{y}}.
$$

### Standard Error (SE)

The standard error (SE) is the standard deviation (of the sampling distribution) of a statistic:
$$
    \operatorname{SE}(X) = \frac{\operatorname{Std}(X)}{\sqrt{n}}.
$$

It is most commonly considered for the mean with the estimator

\begin{align}
\operatorname{SE}(X) &= \operatorname{Std}(X) = \sigma_{\bar{x}}\\
      &= \frac{\sigma_x}{\sqrt{n}}.
\end{align}

### Descriptives statistics with numpy

- Generate 2 random samples: $x \sim N(1.78, 0.1)$ and $y \sim N(1.66, 0.1)$, both of size 10.

- Compute $\bar{x}, \sigma_x, \sigma_{xy}$ (`xbar, xvar, xycov`) using only the `np.sum()` operation.
Explore the `np.` module to find out which numpy functions performs the same computations and compare them (using `assert`) with your previous results.

Caution! By default `np.var()` used the biased estimator (with ddof=0). Set ddof=1 to use unbiased estimator.

```python
n = 10
x = np.random.normal(loc=1.78, scale=.1, size=n)
y = np.random.normal(loc=1.66, scale=.1, size=n)

xbar = np.mean(x)
assert xbar == np.sum(x) / x.shape[0]

xvar = np.var(x, ddof=1)
assert xvar == np.sum((x - xbar) ** 2) / (n - 1)

xycov = np.cov(x, y)
print(xycov)

ybar = np.sum(y) / n
assert np.allclose(xycov[0, 1], np.sum((x - xbar) * (y - ybar)) / (n - 1))
assert np.allclose(xycov[0, 0], xvar)
assert np.allclose(xycov[1, 1], np.var(y, ddof=1))
```

### Descriptives statistics on Iris dataset

**With Pandas**

Columns' means

```python
iris.mean()
```

Columns' std-dev. Pandas normalizes by N-1 by default.

```python
iris.std()
```

**With Numpy**

```python
X = iris[['SepalLength', 'SepalWidth', 'PetalLength', 'PetalWidth']].values
iris.columns
X.mean(axis=0)
```

Columns' std-dev. Numpy normalizes by N by default. Set ddof=1 to normalize by N-1 to get the unbiased estimator.

```python
X.std(axis=0, ddof=1)
```

## Main distributions

### Normal distribution

The normal distribution, noted $\mathcal{N}(\mu, \sigma)$ with parameters: $\mu$ mean (location) and $\sigma>0$ std-dev. Estimators: $\bar{x}$ and $\sigma_{x}$.

The normal distribution, noted $\mathcal{N}$, is useful because of the central limit theorem (CLT) which states that: given certain conditions, the arithmetic mean of a sufficiently large number of iterates of independent random variables, each with a well-defined expected value and well-defined variance, will be approximately normally distributed, regardless of the underlying distribution.

```python
mu = 0 # mean
variance = 2 #variance
sigma = np.sqrt(variance) #standard deviation",
x = np.linspace(mu - 3 * variance, mu + 3 * variance, 100)
_ = plt.plot(x, scipy.stats.norm.pdf(x, mu, sigma))
```

### The Chi-Square distribution

The chi-square or $\chi_n^2$ distribution with $n$ degrees of freedom (df) is the distribution of a sum of the squares of $n$ independent standard normal random variables $\mathcal{N}(0, 1)$. Let $X \sim \mathcal{N}(\mu, \sigma^2)$, then, $Z=(X - \mu)/\sigma \sim \mathcal{N}(0, 1)$, then:

- The squared standard $Z^2 \sim \chi_1^2$ (one df).

- **The distribution of sum of squares** of $n$ normal random variables: $\sum_i^n Z_i^2 \sim \chi_n^2$

The sum of two $\chi^2$ RV with $p$ and $q$ df is a $\chi^2$ RV with $p+q$ df. This is useful when summing/subtracting sum of squares.

The $\chi^2$-distribution is used to model **errors** measured as **sum of squares** or the distribution of the sample **variance**.

### The Fisher's F-distribution

The $F$-distribution, $F_{n, p}$, with $n$ and $p$ degrees of freedom is the ratio of two independent $\chi^2$ variables. Let $X \sim \chi_n^2$ and $Y \sim \chi_p^2$ then:
$$
    F_{n, p} = \frac{X/n}{Y/p}
$$

The $F$-distribution plays a central role in hypothesis testing answering the question: **Are two variances equals?, is the ratio or two errors significantly large ?**.

```python
fvalues = np.linspace(.1, 5, 100)

# pdf(x, df1, df2): Probability density function at x of F.
plt.plot(fvalues, scipy.stats.f.pdf(fvalues, 1, 30), 'b-', label="F(1, 30)")
plt.plot(fvalues, scipy.stats.f.pdf(fvalues, 5, 30), 'r-', label="F(5, 30)")
plt.legend()

# cdf(x, df1, df2): Cumulative distribution function of F.
# ie.
proba_at_f_inf_3 = scipy.stats.f.cdf(3, 1, 30) # P(F(1,30) < 3)

# ppf(q, df1, df2): Percent point function (inverse of cdf) at q of F.
f_at_proba_inf_95 = scipy.stats.f.ppf(.95, 1, 30) # q such P(F(1,30) < .95)
assert scipy.stats.f.cdf(f_at_proba_inf_95, 1, 30) == .95

# sf(x, df1, df2): Survival function (1 - cdf) at x of F.
proba_at_f_sup_3 = scipy.stats.f.sf(3, 1, 30) # P(F(1,30) > 3)
assert  proba_at_f_inf_3 + proba_at_f_sup_3 == 1

# p-value: P(F(1, 30)) < 0.05
low_proba_fvalues = fvalues[fvalues > f_at_proba_inf_95]
plt.fill_between(low_proba_fvalues, 0, scipy.stats.f.pdf(low_proba_fvalues, 1, 30),
                 alpha=.8, label="P < 0.05")
plt.show()
```

### The  Student's $t$-distribution

Let $M \sim \mathcal{N}(0, 1)$ and $V \sim \chi_n^2$. The $t$-distribution, $T_n$, with $n$ degrees of freedom is the ratio:
$$
    T_n = \frac{M}{\sqrt{V/n}}
$$

The distribution of the difference between an estimated parameter and its true (or assumed) value divided by the standard deviation of the estimated parameter (standard error) follow a $t$-distribution. **Is this parameters different from a given value?**

## Hypothesis Testing

**Examples**

- Test a proportion: Biased coin ? 200 heads have been found over 300 flips, is it coins biased ?

- Test the association between two variables.
    * Exemple height and sex: In a sample of 25 individuals (15 females, 10 males), is female height is different from male height ?
    * Exemple age and arterial hypertension: In a sample of 25 individuals is age height correlated with arterial hypertension ?

**Steps**

1. Model the data

2. Fit: estimate the model parameters (frequency, mean, correlation, regression coeficient)

3. Compute a test statistic from model the parameters.

4. Formulate the null hypothesis: What would be the (distribution of the) test statistic if the observations are the result of pure chance.

5. Compute the probability ($p$-value) to obtain a larger value for the test statistic by chance (under the null hypothesis).

### Flip coin: Simplified example

Biased coin ? 2 heads have been found over 3 flips, is it coins biased ?

1. Model the data: number of heads follow a Binomial disctribution.

2. Compute model parameters: N=3, P = the frequency of number of heads over the number of flip: 2/3.

3. Compute a test statistic, same as frequency.

2. Under the null hypothesis the distribution of the number of tail is:

| 1 | 2 | 3 | count #heads |
|---|---|---|--------------|
|   |   |   | 0            |
| H |   |   | 1            |
|   | H |   | 1            |
|   |   | H | 1            |
| H | H |   | 2            |
| H |   | H | 2            |
|   | H | H | 2            |
| H | H | H | 3            |

8 possibles configurations, probabilities of differents values for $p$ are:
$x$ measure the number of success.

- $P(x=0) = 1/8$
- $P(x=1) = 3/8$
- $P(x=2) = 3/8$
- $P(x=3) = 1/8$

```python
plt.figure(figsize=(5, 3))
plt.bar([0, 1, 2, 3], [1/8, 3/8, 3/8, 1/8], width=0.9)
_ = plt.xticks([0, 1, 2, 3], [0, 1, 2, 3])
plt.xlabel("Distribution of the number of head over 3 flip under the null hypothesis")
```

3. Compute the probability ($p$-value) to observe a value larger or equal that 2 under the null hypothesis ?
This probability is the $p$-value:
$$
P(x\geq 2| H_0) = P(x=2) + P(x=3) = 3/8 + 1/8 = 4/8 = 1/2
$$

### Flip coin: Real Example

Biased coin ? 60 heads have been found over 100 flips, is it coins biased ?

1. Model the data: number of heads follow a Binomial disctribution.

2. Compute model parameters: N=100, P=60/100.

3. Compute a test statistic, same as frequency.

4. Compute a test statistic: 60/100.

5. Under the null hypothesis the distribution of the number of tail ($k$) follow the **binomial distribution** of parameters N=100, **P=0.5**:
$$
Pr(X=k|H_0) = Pr(X=k|n=100, p=0.5) = {100 \choose k}0.5^k (1-0.5)^{(100-k)}.
$$

\begin{align*}
P(X=k\geq 60|H_0) &=  \sum_{k=60}^{100}{100 \choose k}0.5^k (1-0.5)^{(100-k)}\\
 &= 1 - \sum_{k=1}^{60}{100 \choose k}0.5^k (1-0.5)^{(100-k)}, \text{the cumulative distribution function.}
\end{align*}

**Use tabulated binomial distribution**

```python
succes = np.linspace(30, 70, 41)
plt.plot(succes, scipy.stats.binom.pmf(succes, 100, 0.5),
         'b-', label="Binomial(100, 0.5)")
upper_succes_tvalues = succes[succes > 60]
plt.fill_between(upper_succes_tvalues, 0,
                 scipy.stats.binom.pmf(upper_succes_tvalues, 100, 0.5),
                 alpha=.8, label="p-value")
_ = plt.legend()

pval = 1 - scipy.stats.binom.cdf(60, 100, 0.5)
print(pval)
```

**Random sampling of the Binomial distribution under the null hypothesis**

```python
sccess_h0 = scipy.stats.binom.rvs(100, 0.5, size=10000, random_state=4)
print(sccess_h0)

pval_rnd = np.sum(sccess_h0 >= 60) / (len(sccess_h0) + 1)
print("P-value using monte-carlo sampling of the Binomial distribution under H0=",
      pval_rnd)
```

###  One sample $t$-test

The one-sample $t$-test is used to determine whether a sample comes from a population with a specific mean. For example you want to test if the average height of a population is $1.75~m$.

#### Assumptions

1. Independence of **residuals** ($\varepsilon_i$). This assumptions **must** be satisfied.
2. Normality of residuals. Approximately normally distributed can be accepted.

Remarks: Although the parent population does not need to be normally distributed, the distribution of the population of sample means, $\overline{x}$, is assumed to be normal. By the central limit theorem, if the sampling of the parent population is independent then the sample means will be approximately normal.

#### 1 Model the data

Assume that height is normally distributed: $X \sim \mathcal{N}(\mu, \sigma)$, ie:

\begin{align}
\text{height}_i &= \text{average height over the population} + \text{error}_i\\
x_i &= \bar{x} + \varepsilon_i
\end{align}

The $\varepsilon_i$ are called the residuals

#### 2 Fit: estimate the model parameters

  $\bar{x}, s_x$ are the estimators of $\mu, \sigma$.

#### 3 Compute a test statistic

In testing the null hypothesis that the population mean is equal to a specified value $\mu_0=1.75$, one uses the statistic:

\begin{align}
 t &= \frac{\text{difference of means}}{\text{std-dev of noise}} \sqrt{n}\\
 t &= \text{effect size} \sqrt{n}\\
 t &= \frac{\bar{x} - \mu_0}{s_x} \sqrt{n}
\end{align}

#### 4 Compute the probability of the test statistic under the null hypotheis. This require to have the distribution of the t statistic under $H_0$.

#### Example

Given the following samples, we will test whether its true mean is 1.75.

Warning, when computing the std or the variance, set `ddof=1`. The default
value, `ddof=0`, leads to the biased estimator of the variance.

```python
x = [1.83,  1.83,  1.73,  1.82,  1.83,  1.73,  1.99,  1.85,  1.68,  1.87]

xbar = np.mean(x)  # sample mean
mu0 = 1.75  # hypothesized value
s = np.std(x, ddof=1)  # sample standard deviation
n = len(x)  # sample size

print(xbar)

tobs = (xbar - mu0) / (s / np.sqrt(n))
print(tobs)
```

The **$p$-value** is the probability to observe a value $t$ more extreme than the observed one $t_{obs}$ under the null hypothesis $H_0$: $P(t > t_{obs} | H_0)$

```python
tvalues = np.linspace(-10, 10, 100)
plt.plot(tvalues, scipy.stats.t.pdf(tvalues, n-1), 'b-', label="T(n-1)")
upper_tval_tvalues = tvalues[tvalues > tobs]
plt.fill_between(upper_tval_tvalues, 0, scipy.stats.t.pdf(upper_tval_tvalues, n-1),
                 alpha=.8, label="p-value")
_ = plt.legend()
```

## Testing pairwise associations

Univariate statistical analysis: explore association betweens pairs of variables.

- In statistics, a **categorical variable** or **factor** is a variable that can take on one of a limited, and usually fixed, number of possible values, thus assigning each individual to a particular group or "category". The levels are the possibles values of the variable. Number of levels = 2: binomial; Number of levels > 2: multinomial. There is no intrinsic ordering to the categories.  For example, gender is a categorical variable having two categories (male and female) and there is no intrinsic ordering to the categories. For example, Sex (Female, Male), Hair color (blonde, brown, etc.).

- An **ordinal variable** is a categorical variable with a clear ordering of the levels. For example: drinks per day (none, small, medium and high).

- A **continuous** or **quantitative variable** $x \in \mathbb{R}$ is one that can take any value in a range of possible values, possibly infinite.  E.g.: salary, experience in years, weight.

**What statistical test should I use?**

See: http://www.ats.ucla.edu/stat/mult_pkg/whatstat/

## Pearson correlation test: test association between two quantitative variables

Test the correlation coefficient of two quantitative variables. The test calculates a Pearson correlation coefficient and the $p$-value for testing non-correlation.

Let $x$ and $y$ two quantitative variables, where $n$ samples were obeserved. The linear correlation coeficient is defined as :

$$r=\frac{\sum_{i=1}^n(x_i-\bar x)(y_i-\bar y)}{\sqrt{\sum_{i=1}^n(x_i-\bar x)^2}\sqrt{\sum_{i=1}^n(y_i-\bar y)^2}}.$$

Under $H_0$, the test statistic $t=\sqrt{n-2}\frac{r}{\sqrt{1-r^2}}$ follow Student distribution with $n-2$ degrees of freedom.

```python
n = 50
x = np.random.normal(size=n)
y = 2 * x + np.random.normal(size=n)

# Compute with scipy
cor, pval = scipy.stats.pearsonr(x, y)
print(cor, pval)
```

## Two sample (Student) $t$-test: compare two means

{width=7cm}

The two-sample $t$-test (Snedecor and Cochran, 1989) is used to determine if two population means are equal. There are several variations on this test.
If data are paired (e.g. 2 measures, before and after treatment for each individual) use the one-sample $t$-test of the difference. The variances of the two samples may be assumed to be equal (a.k.a. homoscedasticity) or unequal (a.k.a. heteroscedasticity).

### Assumptions

1. Independence of **residuals** ($\varepsilon_i$). This assumptions **must** be satisfied.
2. Normality of residuals. Approximately normally distributed can be accepted.
3. Homosedasticity use T-test, Heterosedasticity use Welch t-test.

### 1. Model the data

Assume that the two random variables are normally distributed: $y_1 \sim \mathcal{N}(\mu_{1}, \sigma_{1}), y_2 \sim \mathcal{N}(\mu_{2}, \sigma_2)$.

### 2. Fit: estimate the model parameters

Estimate means and variances: $\bar{y_1}, s^2_{y_1}, \bar{y_2}, s^2_{y_2}$.

### 3. $t$-test

The general principle is

\begin{align}
t &= \frac{\text{difference of means}}{\text{standard dev of error}}\\
  &= \frac{\text{difference of means}}{\text{its standard error}}\\
  &= \frac{\bar{y_1}-\bar{y_2}}{\sqrt{\sum\varepsilon^2}}\sqrt{n-2}\\
&= \frac{\bar{y_1}-\bar{y_2}}{s_{\bar{y_1}-\bar{y_2}}}
\end{align}

Since $y_1$ and $y_2$ are independant:

\begin{align}
s^2_{\bar{y_1}-\bar{y_2}} &= s^2_{\bar{y_1}} + s^2_{\bar{y_2}} = \frac{s^2_{y_1}}{n_1} + \frac{s^2_{y_2}}{n_2}\\
\text{thus}\\
s_{\bar{y_1}-\bar{y_2}} &= \sqrt{\frac{s^2_{y_1}}{n_1} + \frac{s^2_{y_2}}{n_2}}
\end{align}

#### Equal or unequal sample sizes, unequal variances (Welch's $t$-test)

Welch's $t$-test defines the $t$ statistic as

$$
t = \frac{\bar{y_1} - \bar{y_2}}{\sqrt{\frac{s^2_{y_1}}{n_1} + \frac{s^2_{y_2}}{n_2}}}.
$$

To compute the $p$-value one needs the degrees of freedom associated with this variance estimate. It is approximated using the Welch–Satterthwaite equation:

$$
\nu \approx \frac{\left(\frac{s^2_{y_1}}{n_1} + \frac{s^2_{y_2}}{n_2}\right)^2}{\frac{s^4_{y_1}}{n_1^2(n_1-1)} + \frac{s^4_{y_2}}{n_2^2(n_2-1)}}.
$$

#### Equal or unequal sample sizes, equal variances

If we assume equal variance (ie, $s^2_{y_1} = s^2_{y_1} = s^2$), where $s^2$ is an estimator of the common variance of the two samples:

\begin{align}
s^2 &= \frac{s_{y_1}^2(n_1-1)+s_{y_2}^2(n_2-1)}{n_1+n_2-2}\\
    &= \frac{\sum_i^{n_1} (y_{1i} -\bar{y_1})^2 + \sum_j^{n_2} (y_{2j} -\bar{y_2})^2}{(n_1 - 1) + (n_2 - 1)}
\end{align}

then

$$
s_{\bar{y_1}-\bar{y_2}} = \sqrt{\frac{s^2}{n_1} + \frac{s^2}{n_2}} = s \sqrt{\frac{1}{n_1} + \frac{1}{n_2}}
$$

Therefore, the $t$ statistic, that is used to test whether the means are different is:

$$
t = \frac{\bar{y_1} - \bar{y_2}}{s \cdot \sqrt{\frac{1}{n_1}+\frac{1}{n_2}}},
$$

#### Equal sample sizes, equal variances

If we simplify the problem assuming equal samples of size $n_1 = n_2 = n$ we get

\begin{align}
t &= \frac{\bar{y_1} - \bar{y_2}}{s \sqrt{2}} \cdot \sqrt{n}\\
&\approx \text{effect size} \cdot \sqrt{n}\\
&\approx \frac{\text{difference of means}}{\text{standard deviation of the noise}} \cdot \sqrt{n}
\end{align}

#### Example

Given the following two samples, test whether their means are equal using the **standard t-test, assuming equal variance**.

```python
height = np.array([ 1.83,  1.83,  1.73,  1.82,  1.83,  1.73,  1.99,  1.85,  1.68,  1.87,
                    1.66,  1.71,  1.73,  1.64,  1.70,  1.60,  1.79,  1.73,  1.62,  1.77])

grp = np.array(["M"] * 10 + ["F"] * 10)

# Compute with scipy
scipy.stats.ttest_ind(height[grp == "M"], height[grp == "F"], equal_var=True)
```

## ANOVA $F$-test (quantitative ~ categorial (>=2 levels))

Analysis of variance (ANOVA) provides a statistical test of whether or not the means of several (k) groups are equal, and therefore generalizes the $t$-test to more than two groups. ANOVAs are useful for comparing (testing) three or more means (groups or variables) for statistical significance. It is conceptually similar to multiple two-sample $t$-tests, but is less conservative.

Here we will consider one-way ANOVA with one independent variable, ie one-way anova.

[Wikipedia](https://en.wikipedia.org/wiki/F-test):

- Test if any group is on average superior, or inferior, to the others versus the null hypothesis that all four strategies yield the same mean response

- Detect any of several possible differences.

- The advantage of the ANOVA $F$-test is that we do not need to pre-specify which strategies are to be compared, and we do not need to adjust for making multiple comparisons.

- The disadvantage of the ANOVA $F$-test is that if we reject the null hypothesis, we do not know which strategies can be said to be significantly different from the others.

### Assumptions

1. The samples are randomly selected in an independent manner from the k populations.
2. All k populations have distributions that are approximately normal. Check by plotting groups distribution.
3. The k population variances are equal. Check by plotting groups distribution.

### 1. Model the data

Is there a difference in Petal Width in species from iris dataset.
Let $y_1, y_2$ and $y_3$ be Petal Width in three species.

Here we assume (see assumptions) that the three populations were sampled from three random variables that are normally distributed. I.e., $Y_1 \sim N(\mu_1, \sigma_1), Y_2 \sim N(\mu_2, \sigma_2)$ and $Y_3 \sim N(\mu_3, \sigma_3)$.

### 2. Fit: estimate the model parameters

Estimate means and variances: $\bar{y}_i, \sigma_i,\;\; \forall i \in \{1, 2, 3\}$.

### 3. $F$-test

The formula for the one-way ANOVA F-test statistic is

\begin{align}
F &= \frac{\text{Explained variance}}{\text{Unexplained variance}}\\
&=\frac{\text{Between-group variability}}{\text{Within-group variability}} = \frac{s^2_B}{s^2_W}.
\end{align}

The "explained variance", or "between-group variability" is

$$
s^2_B = \sum_i n_i(\bar{y}_{i\cdot} - \bar{y})^2/(K-1),
$$

where $\bar{y}_{i\cdot}$ denotes the sample mean in the $i$th group, $n_i$ is the number of observations in the $i$th group, $\bar{y}$ denotes the overall mean of the data, and $K$ denotes the number of groups.

The "unexplained variance", or "within-group variability" is

$$
s^2_W = \sum_{ij} (y_{ij}-\bar{y}_{i\cdot})^2/(N-K),
$$

where $y_{ij}$ is the $j$th observation in the $i$th out of $K$ groups and $N$ is the overall sample size. This $F$-statistic follows the $F$-distribution with $K-1$ and $N-K$ degrees of freedom under the null hypothesis. The statistic will be large if the between-group variability is large relative to the within-group variability, which is unlikely to happen if the population means of the groups all have the same value.

Note that when there are only two groups for the one-way ANOVA F-test, $F=t^2$ where $t$ is the Student's $t$ statistic.

Iris dataset:

```python
# Group means
means = iris.groupby("Species").mean().reset_index()
print(means)

# Group Stds (equal variances ?)
stds = iris.groupby("Species").std(ddof=1).reset_index()
print(stds)

# Plot groups
ax = sns.violinplot(x="Species", y="SepalLength", data=iris)
ax = sns.swarmplot(x="Species", y="SepalLength", data=iris,
                   color="white")
ax = sns.swarmplot(x="Species", y="SepalLength",  color="black", data=means, size=10)

# ANOVA
lm = smf.ols('SepalLength ~ Species', data=iris).fit()
sm.stats.anova_lm(lm, typ=2) # Type 2 ANOVA DataFrame
```

## Chi-square, $\chi^2$ (categorial ~ categorial)

Computes the chi-square, $\chi^2$, statistic and $p$-value for the hypothesis test of independence of frequencies in the observed contingency table (cross-table). The observed frequencies are tested against an expected contingency table obtained by computing expected frequencies based on the marginal sums under the assumption of independence.

Example: 20 participants: 10 exposed to some chemical product and 10 non exposed (exposed = 1 or 0). Among the 20 participants 10 had cancer 10 not (cancer = 1 or 0). $\chi^2$ tests the association between those two variables.

```python
# Dataset:
# 15 samples:
# 10 first exposed
exposed = np.array([1] * 10 + [0] * 10)
# 8 first with cancer, 10 without, the last two with.
cancer = np.array([1] * 8 + [0] * 10 + [1] * 2)

crosstab = pd.crosstab(exposed, cancer, rownames=['exposed'],
                       colnames=['cancer'])
print("Observed table:")
print("---------------")
print(crosstab)

chi2, pval, dof, expected = scipy.stats.chi2_contingency(crosstab)
print("Statistics:")
print("-----------")
print("Chi2 = %f, pval = %f" % (chi2, pval))
print("Expected table:")
print("---------------")
print(expected)
```

Computing expected cross-table

```python
# Compute expected cross-table based on proportion
exposed_marg = crosstab.sum(axis=0)
exposed_freq = exposed_marg / exposed_marg.sum()

cancer_marg = crosstab.sum(axis=1)
cancer_freq = cancer_marg / cancer_marg.sum()

print('Exposed frequency? Yes: %.2f' % exposed_freq[0],
      'No: %.2f' % exposed_freq[1])
print('Cancer frequency? Yes: %.2f' % cancer_freq[0],
      'No: %.2f' % cancer_freq[1])

print('Expected frequencies:')
print(np.outer(exposed_freq, cancer_freq))

print('Expected cross-table (frequencies * N): ')
print(np.outer(exposed_freq, cancer_freq) * len(exposed))
```

## Non-parametric test of pairwise associations

### Spearman rank-order correlation (quantitative ~ quantitative)

The Spearman correlation is a non-parametric measure of the monotonicity of the relationship between two datasets.

When to use it? Observe the data distribution:
- presence of **outliers**
- the distribution of the residuals is not Gaussian.

Like other correlation coefficients, this one varies between -1 and +1 with 0 implying no correlation. Correlations of -1 or +1 imply an exact monotonic relationship. Positive correlations imply that as $x$ increases, so does $y$. Negative correlations imply that as $x$ increases, $y$ decreases.

```python
np.random.seed(3)

# Age uniform distribution between 20 and 40
age = np.random.uniform(20, 60, 40)

# Systolic blood presure, 2 groups:
# - 15 subjects at 0.05 * age + 6
# - 25 subjects at 0.15 * age + 10
sbp = np.concatenate((0.05 * age[:15] + 6, 0.15 * age[15:] + 10)) + \
    .5 * np.random.normal(size=40)

sns.regplot(x=age, y=sbp)

# Non-Parametric Spearman
cor, pval = scipy.stats.spearmanr(age, sbp)
print("Non-Parametric Spearman cor test, cor: %.4f, pval: %.4f" % (cor, pval))

# "Parametric Pearson cor test
cor, pval = scipy.stats.pearsonr(age, sbp)
print("Parametric Pearson cor test: cor: %.4f, pval: %.4f" % (cor, pval))
```

### Wilcoxon signed-rank test (quantitative ~ cte)

Source: https://en.wikipedia.org/wiki/Wilcoxon_signed-rank_test

The Wilcoxon signed-rank test is a non-parametric statistical hypothesis test used when comparing two related samples, matched samples, or repeated measurements on a single sample to assess whether their population mean ranks differ (i.e. it is a paired difference test). It is equivalent to one-sample test of the difference of paired samples.

It can be used as an alternative to the paired Student's $t$-test, $t$-test for matched pairs, or the $t$-test for dependent samples when the population cannot be assumed to be normally distributed.

When to use it? Observe the data distribution:
- presence of outliers
- the distribution of the residuals is not Gaussian

It has a lower sensitivity compared to $t$-test. May be problematic to use when the sample size is small.

Null hypothesis $H_0$: difference between the pairs follows a symmetric distribution around zero.

```python
n = 20
# Buisness Volume time 0
bv0 = np.random.normal(loc=3, scale=.1, size=n)
# Buisness Volume time 1
bv1 = bv0 + 0.1 + np.random.normal(loc=0, scale=.1, size=n)

# create an outlier
bv1[0] -= 10

# Paired t-test
print(scipy.stats.ttest_rel(bv0, bv1))

# Wilcoxon
print(scipy.stats.wilcoxon(bv0, bv1))
```

### Mann–Whitney $U$ test (quantitative ~ categorial (2 levels))

In statistics, the Mann–Whitney $U$ test (also called the Mann–Whitney–Wilcoxon, Wilcoxon rank-sum test or Wilcoxon–Mann–Whitney test) is a nonparametric test of the null hypothesis that two samples come from the same population against an alternative hypothesis, especially that a particular population tends to have larger values than the other.

It can be applied on unknown distributions contrary to e.g. a $t$-test that has to be applied only on normal distributions, and it is nearly as efficient as the $t$-test on normal distributions.

```python
n = 20
# Buismess Volume group 0
bv0 = np.random.normal(loc=1, scale=.1, size=n)

# Buismess Volume group 1
bv1 = np.random.normal(loc=1.2, scale=.1, size=n)

# create an outlier
bv1[0] -= 10

# Two-samples t-test
print(scipy.stats.ttest_ind(bv0, bv1))

# Wilcoxon
print(scipy.stats.mannwhitneyu(bv0, bv1))
```

## Linear model

{width=5cm}

Given $n$ random samples $(y_i, x_{1i}, \ldots, x_{pi}), \, i = 1, \ldots, n$, the linear regression models the relation between the observations $y_i$ and the independent variables $x_i^p$ is formulated as

$$
    y_i = \beta_0 + \beta_1 x_{1i} + \cdots + \beta_p x_{pi} + \varepsilon_i \qquad i = 1, \ldots, n
$$

- The $\beta$'s are the model parameters, ie, the regression coeficients.
- $\beta_0$ is the intercept or the bias.
- $\varepsilon_i$ are the **residuals**.
- **An independent variable (IV)**. It is a variable that stands alone and isn't changed by the other variables you are trying to measure. For example, someone's age might be an independent variable. Other factors (such as what they eat, how much they go to school, how much television they watch) aren't going to change a person's age. In fact, when you are looking for some kind of relationship between variables you are trying to see if the independent variable causes some kind of change in the other variables, or dependent variables. In Machine Learning, these variables are also called the **predictors**.

- A **dependent variable**. It is something that depends on other factors. For example, a test score could be a dependent variable because it could change depending on several factors such as how much you studied, how much sleep you got the night before you took the test, or even how hungry you were when you took it. Usually when you are looking for a relationship between two things you are trying to find out what makes the dependent variable change the way it does. In Machine Learning this variable is called a **target variable**.

### Assumptions

1. Independence of residuals ($\varepsilon_i$). This assumptions **must** be satisfied
2. Normality of residuals ($\varepsilon_i$). Approximately normally distributed can be accepted.

[Regression diagnostics:  testing the assumptions of linear regression](http://people.duke.edu/~rnau/testing.htm)

### Simple regression: test association between two quantitative variables

Using the dataset "salary", explore the association between the dependant variable (e.g. Salary) and the independent variable (e.g.: Experience is quantitative), considering only non-managers.

```python
df = salary[salary.management == 'N']
```

#### 1. Model the data

Model the data on some **hypothesis** e.g.: salary is a linear function of the experience.

$$
    \text{salary}_i = \beta_0 + \beta~\text{experience}_i + \epsilon_i,
$$

more generally

$$
    y_i = \beta_0 + \beta~x_i + \epsilon_i
$$

This can be rewritten in the matrix form using the design matrix made of values of independant variable and the intercept:

$$
\begin{split}\begin{bmatrix}y_1 \\ y_2 \\ y_3 \\ y_4 \\ y_5  \end{bmatrix}
  =
  \begin{bmatrix}1 & x_1  \\1 & x_2  \\1 & x_3  \\1 & x_4  \\1 & x_5    \end{bmatrix}
  \begin{bmatrix} \beta_0 \\ \beta_1  \end{bmatrix}
  +
  \begin{bmatrix} \epsilon_1 \\ \epsilon_2 \\ \epsilon_3 \\ \epsilon_4 \\ \epsilon_5 \end{bmatrix}\end{split}
$$

- $\beta$: the slope or coefficient or parameter of the model,

- $\beta_0$: the **intercept** or **bias** is the second parameter of the model,

- $\epsilon_i$: is the $i$th error, or residual with $\epsilon \sim \mathcal{N}(0, \sigma^2)$.

The simple regression is equivalent to the Pearson correlation.

#### 2. Fit: estimate the model parameters

The goal it so estimate $\beta$, $\beta_0$ and $\sigma^2$.

Minimizes the **mean squared error (MSE)** or the **Sum squared error (SSE)**. The so-called **Ordinary Least Squares (OLS)** finds $\beta, \beta_0$ that minimizes the $SSE = \sum_i \epsilon_i^2$

$$
SSE = \sum_i(y_i - \beta~x_i - \beta_0)^2
$$

Recall from calculus that an extreme point can be found by computing where the derivative is zero, i.e. to find the intercept, we perform the steps:

$$
\frac{\partial SSE}{\partial \beta_0} = \sum_i(y_i - \beta~x_i - \beta_0) = 0\\
\sum_i y_i = \beta~\sum_i x_i + n~\beta_0\\
n~\bar{y} = n~\beta~\bar{x} + n~\beta_0\\
\beta_0 = \bar{y} - \beta~\bar{x}
$$

To find the regression coefficient, we perform the steps:

$$
\frac{\partial SSE}{\partial \beta} = \sum_i x_i(y_i - \beta~x_i - \beta_0) = 0
$$

Plug in $\beta_0$:

$$
\sum_i x_i(y_i - \beta~x_i - \bar{y} + \beta \bar{x}) = 0\\
\sum_i x_i y_i - \bar{y}\sum_i x_i = \beta \sum_i(x_i - \bar{x})
$$

Divide both sides by $n$:

$$
\frac{1}{n}\sum_i x_i y_i  - \bar{y}\bar{x} = \frac{1}{n}\beta \sum_i(x_i - \bar{x})\\
\beta = \frac{\frac{1}{n}\sum_i x_i y_i  - \bar{y}\bar{x}}{\frac{1}{n}\sum_i(x_i - \bar{x})} = \frac{Cov(x, y)}{Var(x)}.
$$

```python
y, x = df.salary, df.experience
beta, beta0, r_value, p_value, std_err = scipy.stats.linregress(x,y)
print("y = %f x + %f,  r: %f, r-squared: %f,\np-value: %f, std_err: %f"
      % (beta, beta0, r_value, r_value**2, p_value, std_err))

print("Regression line with the scatterplot")
yhat = beta * x  +  beta0 # regression line
plt.plot(x, yhat, 'r-', x, y,'o')
plt.xlabel('Experience (years)')
plt.ylabel('Salary')
plt.show()

print("Using seaborn")
ax = sns.regplot(x="experience", y="salary", data=df)
```

### Multiple regression

#### Theory

Muliple Linear Regression is the most basic supervised learning algorithm.

Given: a set of training data $\{x_1, ... , x_N\}$ with corresponding targets $\{y_1, . . . , y_N\}$.

In linear regression, we assume that the model that generates the data involves only a linear combination of the input variables, i.e.

$$
y_i = \beta_0 + \beta_1 x_{i1} + ... + \beta_P x_{iP} + \varepsilon_i,
$$

or, simplified

$$
y_i  = \beta_0 + \sum_{j=1}^{P-1} \beta_j x_i^j + \varepsilon_i.
$$

Extending each sample with an intercept, $x_i := [1, x_i] \in R^{P+1}$ allows us to use a more general notation based on linear algebra and write it as a simple dot product:

$$
y_i = \mathbf{x}_i^T\mathbf{\beta} + \varepsilon_i,
$$

where $\beta \in R^{P+1}$ is a vector of weights that define the $P+1$ parameters of the model. From now we have $P$ regressors + the intercept.

Using the matrix notation:

$$
\begin{split}\begin{bmatrix}y_1 \\ y_2 \\ y_3 \\ y_4 \\ y_5  \end{bmatrix}
  =
  \begin{bmatrix}1 & x_{11}  & \ldots & x_{1P}\\1 & x_{21} & \ldots & x_{2P}  \\1 & x_{31} & \ldots & x_{3P}  \\1 & x_{41} & \ldots & x_{4P}  \\1 & x_5 & \ldots & x_5    \end{bmatrix}
  \begin{bmatrix} \beta_0 \\ \beta_1 \\ \vdots \\ \beta_P \end{bmatrix}
  +
  \begin{bmatrix} \epsilon_1 \\ \epsilon_2 \\ \epsilon_3 \\ \epsilon_4 \\ \epsilon_5 \end{bmatrix}\end{split}
$$

Let $X = [x_0^T, ... , x_N^T]$ be the ($N \times P+1$) **design matrix** of $N$ samples of $P$ input features with one column of one and let be $y = [y_1, ... , y_N]$ be a vector of the $N$ targets.

$$
y = X \beta + \varepsilon
$$

Minimize the Mean Squared Error MSE loss:

$$
MSE(\beta) = \frac{1}{N}\sum_{i=1}^{N}(y_i - \mathbf{x}_i^T\beta)^2
$$

Using the matrix notation, the **mean squared error (MSE) loss can be rewritten**:

$$
MSE(\beta) = \frac{1}{N}||y - X\beta||_2^2.
$$

The $\beta$ that minimises the MSE can be found by:

\begin{align}
\nabla_\beta \left(\frac{1}{N} ||y - X\beta||_2^2\right) &= 0\\
\frac{1}{N}\nabla_\beta (y - X\beta)^T (y - X\beta) &= 0\\
\frac{1}{N}\nabla_\beta (y^Ty - 2 \beta^TX^Ty + \beta^T X^TX\beta) &= 0\\
    -2X^Ty + 2 X^TX\beta &= 0\\
    X^TX\beta &= X^Ty\\
    \beta &= (X^TX)^{-1} X^Ty,
\end{align}

where $(X^TX)^{-1} X^T$ is a pseudo inverse of $X$.

#### Simulated dataset where:

$$
\begin{split}\begin{bmatrix}y_1 \\ \vdots \\ y_{50}  \end{bmatrix}
  =
  \begin{bmatrix}
  1 & x_{1,1}  & x_{1,2} & x_{1,3} \\
  \vdots & \vdots  & \vdots & \vdots \\
  1 & x_{50,1}  & x_{50,2} & x_{50,3} \\
  \end{bmatrix}
  \begin{bmatrix} 10 \\ 1 \\ 0.5 \\ 0.1 \end{bmatrix}
  +
  \begin{bmatrix} \epsilon_1 \\ \vdots \\ \epsilon_{50} \end{bmatrix}\end{split}
$$

```python
from scipy import linalg
np.random.seed(seed=42)  # make the example reproducible

# Dataset
N, P = 50, 4
X = np.random.normal(size= N * P).reshape((N, P))
## Our model needs an intercept so we add a column of 1s:
X[:, 0] = 1
print(X[:5, :])

betastar = np.array([10, 1., .5, 0.1])
e = np.random.normal(size=N)
y = np.dot(X, betastar) + e
```

#### Fit with `numpy`

Estimate the parameters

```python
Xpinv = linalg.pinv2(X)
betahat = np.dot(Xpinv, y)
print("Estimated beta:\n", betahat)
```

## Linear model with statsmodels

Sources: http://statsmodels.sourceforge.net/devel/examples/

### Multiple regression

#### Interface with statsmodels without formulae  (`sm`)

```python
## Fit and summary:
model = sm.OLS(y, X).fit()
print(model.summary())

# prediction of new values
ypred = model.predict(X)

# residuals + prediction == true values
assert np.all(ypred + model.resid == y)
```

#### Statsmodels with Pandas using formulae (`smf`)

Use `R` language syntax for data.frame. For an additive model:
$y_i = \beta^0 + x_i^1 \beta^1 + x_i^2 \beta^2 + \epsilon_i \equiv$ `y ~ x1 + x2`.

```python
df = pd.DataFrame(np.column_stack([X, y]), columns=['inter', 'x1','x2', 'x3', 'y'])
print(df.columns, df.shape)
# Build a model excluding the intercept, it is implicit
model = smf.ols("y~x1 + x2 + x3", df).fit()
print(model.summary())
```

### Multiple regression with categorical independent variables or factors: Analysis of covariance (ANCOVA)

Analysis of covariance (ANCOVA) is a linear model that blends ANOVA and linear regression. ANCOVA evaluates whether population means of a dependent variable (DV) are equal across levels of a categorical independent variable (IV) often called a treatment, while statistically controlling for the effects of other quantitative or continuous variables that are not of primary interest, known as covariates (CV).

```python
df = salary.copy()

lm = smf.ols('salary ~ experience', df).fit()
df["residuals"] = lm.resid

print("Jarque-Bera normality test p-value %.5f" % \
      sm.stats.jarque_bera(lm.resid)[1])

ax = sns.displot(df, x='residuals', kind="kde", fill=True)
ax = sns.displot(df, x='residuals', kind="kde", hue='management', fill=True)
```

Normality assumption of the residuals can be rejected (p-value < 0.05). There is an efect of the "management" factor, take it into account.

#### One-way AN(C)OVA

- ANOVA: one categorical independent variable, i.e. one factor.

- ANCOVA: ANOVA with some covariates.

```python
oneway = smf.ols('salary ~ management + experience', df).fit()
df["residuals"] = oneway.resid
sns.displot(df, x='residuals', kind="kde", fill=True)
print(sm.stats.anova_lm(oneway, typ=2))
print("Jarque-Bera normality test p-value %.3f" %  \
      sm.stats.jarque_bera(oneway.resid)[1])
```

Distribution of residuals is still not normal but closer to normality.
Both management and experience are significantly associated with salary.

#### Two-way AN(C)OVA

Ancova with two categorical independent variables, i.e. two factors.

```python
twoway = smf.ols('salary ~ education + management + experience', df).fit()

df["residuals"] = twoway.resid
sns.displot(df, x='residuals', kind="kde", fill=True)
print(sm.stats.anova_lm(twoway, typ=2))

print("Jarque-Bera normality test p-value %.3f" %  \
      sm.stats.jarque_bera(twoway.resid)[1])
```

Normality assumtion cannot be rejected. Assume it.
Education, management and experience are significantly associated with salary.

#### Comparing two nested models

`oneway` is nested within `twoway`.  Comparing two nested models tells us if the additional predictors (i.e. `education`) of the full model significantly decrease the residuals. Such comparison can be done using an $F$-test on residuals:

```python
print(twoway.compare_f_test(oneway))  # return F, pval, df
```

twoway is significantly better than one way

#### Factor coding

See http://statsmodels.sourceforge.net/devel/contrasts.html

By default Pandas use "dummy coding". Explore:

```python
print(twoway.model.data.param_names)
print(twoway.model.data.exog[:10, :])
```

#### Contrasts and post-hoc tests

```python
# t-test of the specific contribution of experience:
ttest_exp = twoway.t_test([0, 0, 0, 0, 1])
ttest_exp.pvalue, ttest_exp.tvalue
print(ttest_exp)

# Alternatively, you can specify the hypothesis tests using a string
twoway.t_test('experience')

# Post-hoc is salary of Master different salary of Ph.D?
# ie. t-test salary of Master = salary of Ph.D.
print(twoway.t_test('education[T.Master] = education[T.Ph.D]'))
```

## Multiple comparisons

```python
np.random.seed(seed=42)  # make example reproducible

# Dataset
n_samples, n_features = 100, 1000
n_info = int(n_features/10)  # number of features with information
n1, n2 = int(n_samples/2), n_samples - int(n_samples/2)
snr = .5
Y = np.random.randn(n_samples, n_features)
grp = np.array(["g1"] * n1 + ["g2"] * n2)

# Add some group effect for Pinfo features
Y[grp=="g1", :n_info] += snr

#
import scipy.stats as stats
import matplotlib.pyplot as plt
tvals, pvals = np.full(n_features, np.NAN), np.full(n_features, np.NAN)
for j in range(n_features):
    tvals[j], pvals[j] = stats.ttest_ind(Y[grp=="g1", j], Y[grp=="g2", j],
                                         equal_var=True)

fig, axis = plt.subplots(3, 1, figsize=(9, 9))#, sharex='col')

axis[0].plot(range(n_features), tvals, 'o')
axis[0].set_ylabel("t-value")

axis[1].plot(range(n_features), pvals, 'o')
axis[1].axhline(y=0.05, color='red', linewidth=3, label="p-value=0.05")
#axis[1].axhline(y=0.05, label="toto", color='red')
axis[1].set_ylabel("p-value")
axis[1].legend()

axis[2].hist([pvals[n_info:], pvals[:n_info]],
    stacked=True, bins=100, label=["Negatives", "Positives"])
axis[2].set_xlabel("p-value histogram")
axis[2].set_ylabel("density")
axis[2].legend()

plt.tight_layout()
```

Note that under the null hypothesis the distribution of the *p*-values is uniform.

Statistical measures:

- **True Positive (TP)** equivalent to a hit. The test correctly concludes the presence of an effect.

- True Negative (TN). The test correctly concludes the absence of an effect.

- **False Positive (FP)** equivalent to a false alarm, **Type I error**. The test improperly concludes the presence of an effect. Thresholding at $p\text{-value} < 0.05$ leads to 47 FP.

- False Negative (FN) equivalent to a miss, Type II error. The test improperly concludes the absence of an effect.

```python
P, N = n_info, n_features - n_info  # Positives, Negatives
TP = np.sum(pvals[:n_info ] < 0.05)  # True Positives
FP = np.sum(pvals[n_info: ] < 0.05)  # False Positives
print("No correction, FP: %i (expected: %.2f), TP: %i" % (FP, N * 0.05, TP))
```

### Bonferroni correction for multiple comparisons

The Bonferroni correction is based on the idea that if an experimenter is testing $P$ hypotheses, then one way of maintaining the familywise error rate (FWER) is to test each individual hypothesis at a statistical significance level of $1/P$ times the desired maximum overall level.

So, if the desired significance level for the whole family of tests is $\alpha$ (usually 0.05), then the Bonferroni correction would test each individual hypothesis at a significance level of $\alpha/P$. For example, if a trial is testing $P = 8$ hypotheses with a desired $\alpha = 0.05$, then the Bonferroni correction would test each individual hypothesis at $\alpha = 0.05/8 = 0.00625$.

```python
import statsmodels.sandbox.stats.multicomp as multicomp

_, pvals_fwer, _, _  = multicomp.multipletests(pvals, alpha=0.05,
                                               method='bonferroni')
TP = np.sum(pvals_fwer[:n_info ] < 0.05)  # True Positives
FP = np.sum(pvals_fwer[n_info: ] < 0.05)  # False Positives
print("FWER correction, FP: %i, TP: %i" % (FP, TP))
```

### The False discovery rate (FDR) correction for multiple comparisons

FDR-controlling procedures are designed to control the expected proportion of rejected null hypotheses that were incorrect rejections ("false discoveries"). FDR-controlling procedures provide less stringent control of Type I errors compared to the familywise error rate (FWER) controlling procedures (such as the Bonferroni correction), which control the probability of at least one Type I error. Thus, FDR-controlling procedures have greater power, at the cost of increased rates of Type I errors.

```python
_, pvals_fdr, _, _  = multicomp.multipletests(pvals, alpha=0.05,
                                               method='fdr_bh')
TP = np.sum(pvals_fdr[:n_info ] < 0.05)  # True Positives
FP = np.sum(pvals_fdr[n_info: ] < 0.05)  # False Positives

print("FDR correction, FP: %i, TP: %i" % (FP, TP))
```


# Density-Based Clustering

**CS5483 Data Warehousing and Data Mining**
$\def\abs#1{\left\lvert #1 \right\rvert}
\def\Set#1{\left\{ #1 \right\}}
\def\mc#1{\mathcal{#1}}
\def\M#1{\boldsymbol{#1}}
\def\R#1{\mathsf{#1}}
\def\RM#1{\boldsymbol{\mathsf{#1}}}
\def\op#1{\operatorname{#1}}
\def\E{\op{E}}
\def\d{\mathrm{\mathstrut d}}
$
___

$\def\abs#1{\left\lvert #1 \right\rvert}
\def\Set#1{\left\{ #1 \right\}}
\def\mc#1{\mathcal{#1}}
\def\M#1{\boldsymbol{#1}}
\def\R#1{\mathsf{#1}}
\def\RM#1{\boldsymbol{\mathsf{#1}}}
\def\op#1{\operatorname{#1}}
\def\E{\op{E}}
\def\d{\mathrm{\mathstrut d}}
$

```python
%reset -f
%matplotlib inline
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn import datasets, preprocessing
from sklearn.pipeline import make_pipeline
from sklearn.cluster import DBSCAN, OPTICS
# produce vector inline graphics
from IPython.display import set_matplotlib_formats
set_matplotlib_formats('svg')
from ipywidgets import interact
import ipywidgets as widgets
from scipy.optimize import linear_sum_assignment
import seaborn as sns
```

## DBSCAN with scikit-learn

[DBSCAN (Density-based spatial clustering of applications with noise)](https://en.wikipedia.org/wiki/DBSCAN) is a clustering algorithm that identifies clusters as regions of densely populated instances. It makes use of two main parameters $\varepsilon$ and $\op{MinPts}$ to define how dense is dense. It first defines a point $\M{p}\in D$ in the dataset as a *core point* if it satisfies

\begin{align}
|D\cap N_{\varepsilon}(\M{p})|&\geq \op{MinPts} \quad \text{where}\\
N_{\varepsilon}(\M{p})&:= \Set{\M{q}\in D| \op{dist}(\M{p},\M{q})\leq \varepsilon}.
\end{align}

The core points form the *pillars* of the clusters to generate. More precisely, clusters are grown from core points to points in their neighborhood $N_{\varepsilon}(\M{p})$, referred to as *density-reachable* points.

- A point with no cluster assignment, i.e., not *density-reachable* from a core point, regarded as noise.
- A non-core point reachable from a core-point is called a *border point*.

We will create synthetic datasets using the [sample generators](https://scikit-learn.org/stable/modules/classes.html#samples-generator) of `sklearn`. In particular, we first create spherical clusters using [`sklearn.datasets.make_blobs`](https://scikit-learn.org/stable/modules/generated/sklearn.datasets.make_blobs.html#sklearn.datasets.make_blobs):

```python
def XY2df(X, Y):
    '''Return a DataFrame for 2D data with 2 input features X and a target Y.'''
    df = pd.DataFrame(columns=['feature1', 'feature2', 'target'])
    df['target'] = Y
    df[['feature1', 'feature2']] = X
    return df
```

```python
@interact
def generate_blobs(n_samples=widgets.IntSlider(value=200,
                                               min=10,
                                               max=1000,
                                               continuous_update=False),
                   centers=widgets.IntSlider(value=3,
                                             min=1,
                                             max=4,
                                             continuous_update=False),
                   cluster_std=widgets.FloatSlider(value=0.5,
                                                   min=0,
                                                   max=5,
                                                   step=0.1,
                                                   continuous_update=False)):
    df = XY2df(*datasets.make_blobs(n_samples=n_samples,
                                    centers=centers,
                                    cluster_std=cluster_std,
                                    random_state=0))
    fig, ax = plt.subplots()
    ax.set_aspect('equal')
    sns.scatterplot(data=df, x='feature1', y='feature2', hue='target', ax=ax)
```

We will use the dataset `df_spherical` created with the default parameters specified above:

```python
df_spherical = XY2df(*datasets.make_blobs(
    n_samples=200, centers=3, cluster_std=0.5, random_state=0))
```

To create non-spherical clusters, one way is to use [`sklearn.datasets.make_circle`](https://scikit-learn.org/stable/modules/generated/sklearn.datasets.make_circles.html).

```python
df_nonspherical = XY2df(*datasets.make_circles(
    n_samples=200, factor=.1, noise=.1, random_state=0))
```

**Exercise** Complete the following code by assigning `X` and `Y` to the respective arrays of input features and target generated using [`sklearn.datasets.make_circle`](https://scikit-learn.org/stable/modules/generated/sklearn.datasets.make_circles.html). Set `random_state=0` and use the parameters `n_samples`, `factor`, and `noise` specified by the widgets.

```python
import seaborn as sns

@interact
def generate_circles(n_samples=widgets.IntSlider(value=200,
                                                 min=10,
                                                 max=1000,
                                                 continuous_update=False),
                     factor=widgets.FloatSlider(value=0.1,
                                                min=0,
                                                max=0.99,
                                                step=0.01,
                                                continuous_update=False),
                     noise=widgets.FloatSlider(value=0.1,
                                               min=0,
                                               max=1,
                                               step=0.1,
                                               continuous_update=False)):
    df = pd.DataFrame(columns=['feature1', 'feature2', 'target'])
    # YOUR CODE HERE
    raise NotImplementedError()
    df['target'] = Y
    df[['feature1', 'feature2']] = X
    fig, ax = plt.subplots()
    ax.set_aspect('equal')
    sns.scatterplot(data=df, x='feature1', y='feature2', hue='target', ax=ax)
```

To normalize the features followed by [DBSCAN](https://scikit-learn.org/stable/modules/generated/sklearn.cluster.DBSCAN.html), we create a pipeline as follows:

```python
from sklearn.cluster import DBSCAN

dbscan_minmax_normalized = make_pipeline(preprocessing.MinMaxScaler(),
                                         DBSCAN(eps=0.3, min_samples=3))
dbscan_minmax_normalized
```

    Pipeline(steps=[('minmaxscaler', MinMaxScaler()),
                    ('dbscan', DBSCAN(eps=0.3, min_samples=3))])

To generate the clustering solution, we can again use the `fit_predict` method as follows:

```python
feature1, feature2 = df_spherical.columns[0:2]

cluster_labels = dbscan_minmax_normalized.fit_predict(
    df_spherical[[feature1, feature2]])

plt.figure(figsize=(10, 5))
plt.subplot(121)
plt.scatter(df_spherical[feature1], df_spherical[feature2], c=cluster_labels)
plt.title("Cluster assignment")
plt.xlabel(feature1)
plt.ylabel(feature2)
plt.subplot(122)
plt.scatter(df_spherical[feature1],
            df_spherical[feature2],
            c=df_spherical['target'])
plt.title("Class (ground truth)")
plt.xlabel(feature1)
plt.show()
```

**Exercise** The clustering solution above is not correct because the points at the top (`feature2 > 2`) have the same cluster label but they may belong to two different classes. Explain how we should change the parameters, `eps = 0.3` and `min_samples = 3`, to improve the solution. Note that the pairwise distance of points in different clusters appear to be larger than 1.

YOUR ANSWER HERE

**Exercise** Complete the following code so to apply DBSCAN to the different datasets with different choices of parameters.

```python
@interact(cluster_shape=['spherical', 'non-spherical'],
          eps=widgets.FloatSlider(value=0.3,
                                  min=0.01,
                                  max=1,
                                  step=0.01,
                                  continuous_update=False),
          min_samples=widgets.IntSlider(value=3,
                                        min=1,
                                        max=10,
                                        continuous_update=False))
def cluster_regions_dbscan(cluster_shape, eps, min_samples):
    df = {
        'spherical': df_spherical,
        'non-spherical': df_nonspherical
    }[cluster_shape]
    feature1, feature2 = df.columns[0:2]
    # YOUR CODE HERE
    raise NotImplementedError()
    plt.figure(figsize=(10, 5))
    plt.subplot(121)
    plt.scatter(df[feature1], df[feature2], c=cluster_labels)
    plt.title("Cluster assignment")
    plt.xlabel(feature1)
    plt.ylabel(feature2)
    plt.subplot(122)
    plt.scatter(df[feature1], df[feature2], c=df['target'])
    plt.title("Class (ground truth)")
    plt.xlabel(feature1)
    plt.show()
```

**Exercise** It is possible tune `eps` to cluster the generated datasets correctly with `min_samples = 1`, which simplifies to the single-linkage algorithm?

YOUR ANSWER HERE

## OPTICS with Weka

We will use the package `optics_dbScan` in Weka for the density-based clustering algorithms. The package can be installed from Weka GUI -> `tools` -> `Package manager`.

Instead of using the Weka package manager to install `optics_dbScan`, you can also install it by creating a symbolic link as follows and from Weka from JupyterHub `Xpra desktop service`:

```python
if input('Create symlink to Weka packages for this notebook? [Y/n]').lower() != 'n':
    !mkdir -p ~/wekafiles/packages/
    !ln -s /home/course/cs5483/wekafiles/packages/optics_dbScan ~/wekafiles/packages/
```

Open the explorer interface and load the `iris.arff` dataset (not `iris.2D.arff`). Under the `Cluster` panel:

1. Choose `OPTICS` as the `Clusterer`.
1. Choose `Use training set` as the `Cluster mode`.
1. Ignore the `class` attribute using the `Ignore attributes` button.
1. Click `Start`.

The OPTICS Visualizer will appear. The `Table` tab shows the list of data points in the order visited by the algorithm:

[OPTICS (Ordering points to identify the clustering structure)](https://en.wikipedia.org/wiki/OPTICS_algorithm) starts at an arbitrary point and visit other points based on a priority queue that prioritize a point $\M{q}$ with smaller

\begin{align}
\op{reachability-distance}(\M{q}) &:= \max \Set{\op{dist}(\M{p}, \M{q}), \op{core-distance}(\M{p})} \quad \text{where}\\
\op{core-distance}(\M{p}) &:= \min\Set{0\leq \varepsilon' \leq \varepsilon| |D\cap N_{\varepsilon'}(\M{p})|\geq \op{MinPts}}
\end{align}

and $\M{p}$ is a core point that is visited before $\M{q}$ and that yields the smallest reachability distance, but with the following exception:

- If a point $\M{p}$ is not a core point, its core distance is undefined.
- If no such point $\M{p}$ exists for $\M{q}$, the reachability distance of $\M{q}$ is undefined.

**Exercise** The reachability distance is always undefined for the first point visited. Why?

For the first point $\M{q}$ visited, there is no last visited point $\M{p}$ and so both the reachability distance is undefined.

The `Graph` tab shows the stem plots of core and reachability distances. We need to increase the `Vertical adjustment` in the `General Settings` panel to see the variations more clearly:

**Exercise** Note that, by definition, $\op{reachability-distance}(\M{q}) \geq \op{core-distance(\M{p})}$. Does this condition hold for the last two visited points $\M{p}$ and $\M{q}$? Why or Why not?

The condition does not hold for the last two visited points. This is not an error in the implementation, however, because $\M{p}$ may not be the core point that reach $\M{q}$. In other words, there may be some core points, not the second last visited point, closer to $\M{q}$.

Change the `General Settings` to give the reachability plot below:

The above stem plot is called the reachability plot. To obtain a cluster from the plot,

1. specify a threshold to clip the reachability distance from above, and
1. identify a cluster as a "valley" of consecutively visited points with reachability distances below the threshold, except for 1. the first point which should have a reachability distance above the threshold.
1. All other points not assigned to a cluster are regarded as noise.

**Exercise** Assign to `eps_cl` a threshold value that can results in $2$ clusters and no noise points.

*Hint*: You can see the reachability distance of a stem in the reachability plot by hovering the mouse over the stem.

```python
# YOUR CODE HERE
raise NotImplementedError()
eps_cl
```

```python
# tests
```

**Exercise** Assign to `eps_cl` a threshold value that results in $3$ clusters. In particular, choose the threshold value that leads to as few noise points as possible.

```python
# YOUR CODE HERE
raise NotImplementedError()
eps_cl
```

```python
# tests
```

**Exercise** To evaluate the 3 clusters obtained from a particular threshold using an extrinsic measure, run DBSCAN with
- the parameter `epsilon` set to the threshold you obtained and
- Cluster mode set to `Classes to clusters evaluation`.

Assign to `error_rate` the fraction of incorrectly classified instances and `miss_rate` as the fraction of instances not assigned to a cluster.

```python
# YOUR CODE HERE
raise NotImplementedError()
error_rate, miss_rate
```

```python
# tests
```

Note that noise points in DBSCAN may be clusters at a different level of density. The reachability plot of OPTICS can help identify the thresholds for clusters with different densities.