# Nonparametric estimatio of Doppler function

```python
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import scipy.stats as ss
import sympy as sp

sns.set_context('notebook')
%matplotlib inline
```

# Doppler function

$$r\left(x\right)=\sqrt{x\left(1-x\right)}\sin\left(\frac{1.2\pi}{x+.05}\right),\quad x\in\left[0,1\right]$$

```python
x = np.linspace(.01, .99, num=1e3)
doppler = lambda x : np.sqrt(x * (1 - x)) * np.sin(1.2 * np.pi / (x + .05))

plt.plot(x, doppler(x))
plt.show()
```

# Derivative of Doppler function

```python
from sympy.utilities.lambdify import lambdify
from IPython.display import display, Math, Latex

u = sp.Symbol('u')

sym_doppler = lambda x : (x * (1 - x))**.5 * sp.sin(1.2 * sp.pi / (x + .05))
d_doppler = sym_doppler(u).diff()
dd_doppler = sym_doppler(u).diff(n=2)

display(Math(sp.latex(d_doppler)))

d_doppler = np.vectorize(lambdify(u, d_doppler))
dd_doppler = np.vectorize(lambdify(u, dd_doppler))

plt.plot(x, d_doppler(x))
plt.show()
```

# Left and right truncated exponentials

Right truncated:
$$f\left(x\right)=\frac{e^{-x/\lambda}/\lambda}{1-e^{-b/\lambda}},\quad F\left(x\right)=\frac{1-e^{-x/\lambda}}{1-e^{-b/\lambda}},\quad F^{-1}\left(x\right)=-\lambda\log\left(1-x\left(1-e^{-b/\lambda}\right)\right)$$

Left truncated:
$$f\left(x\right)=\frac{e^{x/\lambda}/\lambda}{e^{b/\lambda}-1},\quad F\left(x\right)=\frac{1-e^{x/\lambda}}{1-e^{b/\lambda}},\quad F^{-1}\left(x\right)=\lambda\log\left(1-x\left(1-e^{b/\lambda}\right)\right)$$

```python
def f_rtexp(x, lmbd=1, b=1):
    return np.exp(-x / lmbd) / lmbd / (1 - np.exp(-b / lmbd))

def f_ltexp(x, lmbd=1, b=1):
    return np.exp(x / lmbd) / lmbd / (np.exp(b / lmbd) - 1)

def right_trunc_exp(lmbd=1, b=1, size=1000):
    X = np.sort(np.random.rand(size))
    return - lmbd * np.log(1 - X * (1 - np.exp(-b / lmbd)))

def left_trunc_exp(lmbd=1, b=1, size=1000):
    X = np.sort(np.random.rand(size))
    return lmbd * np.log(1 - X * (1 - np.exp(b / lmbd)))

# Equivalent using SciPy:
# Y = ss.truncexpon.rvs(1, size=1000)

lmbd = .2

Y1 = right_trunc_exp(lmbd=lmbd)
Y2 = left_trunc_exp(lmbd=lmbd)
density1 = ss.gaussian_kde(Y1)
density2 = ss.gaussian_kde(Y2)

U = np.linspace(0, 1, num=1e3)
```

## Draw the densitites

```python
fig = plt.figure(figsize=(15, 5))

plt.subplot(1, 2, 1)
plt.hist(Y1, normed=True, bins=20, label='Histogram')
plt.plot(U, f_rtexp(U, lmbd=lmbd), lw=4, color=[0, 0, 0], label='True density')
plt.plot(U, density1(U), lw=4, color='red', label='Kernel density')
plt.legend()
plt.title('Right truncated')

plt.subplot(1, 2, 2)
plt.hist(Y2, normed=True, bins=20, label='Histogram')
plt.plot(U, f_ltexp(U, lmbd=lmbd), lw=4, color=[0, 0, 0], label='True density')
plt.plot(U, density2(U), lw=4, color='red', label='Kernel density')
plt.legend()
plt.title('Left truncated')

plt.show()
```

# Kernels

Truncated (Uniform): $k_{0}\left(u\right)=\frac{1}{2}1\left(\left|u\right|\leq1\right)$

Epanechnikov: $k_{1}\left(u\right)=\frac{3}{4}\left(1-u^{2}\right)1\left(\left|u\right|\leq1\right)$

Biweight: $k_{2}\left(u\right)=\frac{15}{16}\left(1-u^{2}\right)^{2}1\left(\left|u\right|\leq1\right)$

Triweight: $k_{2}\left(u\right)=\frac{35}{36}\left(1-u^{2}\right)^{3}1\left(\left|u\right|\leq1\right)$

Gaussian: $k_{\phi}\left(u\right)=\frac{1}{\sqrt{2\pi}}\exp\left(-\frac{1}{2}u^2\right)$

```python
def indicator(x):
    return np.asfarray((np.abs(x) <= 1.) & (np.abs(x) >= 0.))

def kernel(x, ktype='Truncated'):
    if ktype == 'Truncated':
        return .5 * indicator(x)
    if ktype == 'Epanechnikov':
        return 3./4. * (1 - x**2) * indicator(x)
    if ktype == 'Biweight':
        return 15./16. * (1 - x**2)**2 * indicator(x)
    if ktype == 'Triweight':
        return 35./36. * (1 - x**2)**3 * indicator(x)
    if ktype == 'Gaussian':
        return 1./np.sqrt(2. * np.pi) * np.exp(- .5 * x**2)

def roughness(ktype='Truncated'):
    if ktype == 'Truncated':
        return 1./2.
    if ktype == 'Epanechnikov':
        return 3./5.
    if ktype == 'Biweight':
        return 5./7.
    if ktype == 'Triweight':
        return 350./429.
    if ktype == 'Gaussian':
        return np.pi**(-.5)/2.

def sigmak(ktype='Truncated'):
    if ktype == 'Truncated':
        return 1./3.
    if ktype == 'Epanechnikov':
        return 1./5.
    if ktype == 'Biweight':
        return 1./7.
    if ktype == 'Triweight':
        return 1./9.
    if ktype == 'Gaussian':
        return 1.

x = np.linspace(0., 2., 100)

names = ['Truncated', 'Epanechnikov', 'Biweight', 'Triweight', 'Gaussian']

for name in names:
    plt.plot(x, kernel(x, ktype=name), label=name, lw=2)
plt.legend()
plt.show()
```

# Nadaraya-Watson (NW) or local constant estimator

## Local weighting

For each observed data $X$ ($N$-vector) and grid $U$ ($M$-vector) this function returns $N\times M$-matrix of weights

```python
def weight(U, X, h=.1, ktype='Truncated'):
    # X - N-array
    # U - M-array

    # XmU - M*N-array
    XmU = (X - np.atleast_2d(U).T) / h
    # K - M*N-array
    K = kernel(XmU, ktype)

    # K.sum(1) - M-array
    # K.T - N*M-array
    # K.T / K.sum(1) - N*M-array
    return  (K.T / K.sum(1)).T
```

## Nadaraya-Watson (NW)

$$\hat{m}\left(x\right)=\frac{\sum_{i=1}^{n}k\left(\frac{X_{i}-x}{h}\right)Y_{i}}{\sum_{i=1}^{n}k\left(\frac{X_{i}-x}{h}\right)}$$

```python
def NW(U, X, Y, h=.1, ktype='Truncated'):
    return np.dot(weight(U, X, h, ktype), Y)
```

## Generate data

$$Y_{i}=m\left(X_{i}\right)+\epsilon_{i},\quad\epsilon_{i}\sim NID\left(0,\sigma=0.1\right)$$

```python
def generate_data(N=1000, M=500, lmbd=1, trunc='left'):

    if trunc == 'left':
        X = left_trunc_exp(lmbd=lmbd, size=N)
    if trunc == 'right':
        X = right_trunc_exp(lmbd=lmbd, size=N)

    e = np.random.normal(0, .1, N)
    Y = doppler(X) + e
    U = np.linspace(.01, .99, M)
    return X, Y, U
```

## Perform estimation and plot the results

```python
X, Y, U = generate_data()

# Nadaraya-Watson estimator
Yhat = NW(U, X, Y, h=.05, ktype='Truncated')

fig = plt.figure(figsize=(10, 6))
plt.plot(U, doppler(U), lw=2, color='blue', label='True')
plt.plot(U, Yhat, lw=2, color='red', label='Fitted')
plt.scatter(X, Y, s=15, lw=.5, facecolor='none', label='Realized')
plt.xlim([0, 1])
plt.xlabel('X')
plt.ylabel('Y')
plt.legend()
plt.show()
```

## Bias correction

For bias computation we take the density of $X$ and two derivatives of conditional mean $m(x)$ as known. In practice, they have to be estimated.

```python
def fx(x, lmbd=1, b=1):
    return sp.exp(-x / lmbd) / lmbd / (1 - sp.exp(-b / lmbd))

dfx = fx(u).diff()
fx = np.vectorize(lambdify(u, fx(u)))
dfx = np.vectorize(lambdify(u, dfx))

def bias(U, etype='NW', h=.05, ktype='Gaussian'):
    if etype == 'NW':
        bias = .5 * dd_doppler(U) + d_doppler(U) * dfx(U) / fx(U)
    if etype == 'LL':
        bias = .5 * dd_doppler(U) * fx(U)
    return bias * h**2 * sigmak(ktype)

h = .05
ktype = 'Gaussian'
fig = plt.figure(figsize=(15, 6))

X, Y, U = generate_data()
Yhat = NW(X, X, Y, h=h, ktype=ktype)
Ynobias = Yhat - bias(X, etype='NW', h=h, ktype=ktype)

plt.plot(X, doppler(X), lw=2, color='blue', label='True')
plt.plot(X, Yhat, lw=2, color='red', label='Fitted')
plt.scatter(X, Y, s=15, lw=.5, facecolor='none', label='Realized')
plt.plot(X, Ynobias, lw=2, color='green', label='No Bias')
plt.xlim([0, 1])
plt.xlabel('X')
plt.ylabel('Y')
plt.legend()
plt.show()
```

# Local Linear (LL) estimator

$$\left(\begin{array}{c}
\hat{\alpha}\left(x\right)\\
\hat{\beta}\left(x\right)
\end{array}\right)=\left(\sum_{i=1}^{n}k_{i}\left(x\right)Z_{i}\left(x\right)Z_{i}\left(x\right)^{\prime}\right)^{-1}\sum_{i=1}^{n}k_{i}\left(x\right)Z_{i}\left(x\right)Y_{i}$$

$$\left(\begin{array}{c}
\hat{\alpha}\left(x\right)\\
\hat{\beta}\left(x\right)
\end{array}\right)
=\left(Z\left(x\right)^{\prime}K\left(x\right)Z\left(x\right)\right)^{-1}Z\left(x\right)^{\prime}K\left(x\right)Y$$

$K(x)$ - $N\times N$

$Z(x)$ - $N\times 2$

$Y$ - $N\times 1$

```python
def LL(U, X, Y, h=.1, ktype='Truncated'):
    # X - N-array
    # U - M-array

    # K - M*N-array
    W = weight(U, X, h, ktype)
    alpha = np.empty(U.shape[0])
    beta = np.empty(U.shape[0])

    for i in range(U.shape[0]):
        # N*N-array
        K = np.diag(W[i])
        # N-array
        Z1 = (X - U[i]) / h
        Z0 = np.ones(Z1.shape)
        # 2*N-array
        Z = np.vstack([Z0, Z1]).T

        # 2*2-array
        A = np.dot(Z.T, np.dot(K, Z))
        # 2-array
        B = np.dot(Z.T, np.dot(K, Y))
        # 2-array
        coef = np.dot(np.linalg.inv(A), B)

        alpha[i] = coef[0]
        beta[i] = coef[1]

    return alpha, beta
```

## Perform estimation and plot the results

```python
X, Y, U = generate_data()

Yhat, dYhat = LL(U, X, Y, h=.05, ktype='Gaussian')

fig = plt.figure(figsize=(15, 6))

plt.subplot(1, 2, 1)
plt.plot(U, doppler(U), lw=2, color='blue', label='True')
plt.plot(U, Yhat, lw=2, color='red', label='Fitted')
plt.scatter(X, Y, s=15, lw=.5, facecolor='none', label='Realized')
plt.xlim([0, 1])
plt.xlabel('X')
plt.ylabel('Y')
plt.legend()
plt.title('Doppler function')

plt.subplot(1, 2, 2)
plt.plot(U, d_doppler(U), lw=2, color='blue', label='True')
plt.plot(U, dYhat, lw=2, color='red', label='Fitted')
plt.xlim([0, 1])
plt.xlabel('X')
plt.ylabel('Y')
plt.legend()
plt.title('Doppler function derivative')

plt.show()
```

# Comparison for different DGP of X

```python
X1, Y1, U = generate_data(lmbd=.1, trunc='left')
X2, Y2, U = generate_data(lmbd=.1, trunc='right')

ktype = 'Gaussian'
h = .05

Y1hat = NW(U, X1, Y1, h=h, ktype=ktype)
Y2hat = NW(U, X2, Y2, h=h, ktype=ktype)

fig = plt.figure(figsize=(15, 10))

plt.subplot(2, 2, 1)
plt.hist(X1, normed=True, bins=20, label='Histogram')
plt.ylabel('X1')

plt.subplot(2, 2, 2)
plt.hist(X2, normed=True, bins=20, label='Histogram')
plt.ylabel('X2')

plt.subplot(2, 2, 3)
plt.plot(U, doppler(U), lw=2, color='blue', label='True')
plt.plot(U, Y1hat, lw=2, color='red', label='Fitted')
plt.scatter(X1, Y1, s=15, lw=.5, facecolor='none', label='Realized')
plt.xlim([0, 1])
plt.xlabel('X1')
plt.ylabel('Y1')
plt.legend()

plt.subplot(2, 2, 4)
plt.plot(U, doppler(U), lw=2, color='blue', label='True')
plt.plot(U, Y2hat, lw=2, color='red', label='Fitted')
plt.scatter(X2, Y2, s=15, lw=.5, facecolor='none', label='Realized')
plt.xlim([0, 1])
plt.xlabel('X2')
plt.ylabel('Y2')
plt.legend()

plt.show()
```

# Conditional variance and confidence intervals

## Leave-one-out errors

```python
def error(Y, X, h, ktype):
    ehat = np.empty(X.shape)

    for i in range(X.shape[0]):
        ehat[i] = Y[i] - NW(X[i], np.delete(X, i), np.delete(Y, i), h=h, ktype=ktype)

    return np.array(ehat)
```

## Estimate variance

```python
N = 500
X, Y, U = generate_data(N=N, lmbd=.2)

h = .05
ktype = 'Epanechnikov'

Yhat = NW(U, X, Y, h=h, ktype=ktype)

ehat = error(Y, X, h, ktype)
sigma2hat = NW(U, X, ehat**2, h=.1, ktype=ktype)
fxhat = ss.gaussian_kde(X)(U)
V2hat = roughness(ktype) * sigma2hat / fxhat / N / h
shat = V2hat**.5
```

## Plot the results

```python
fig = plt.figure(figsize = (10, 10))

plt.subplot(3, 1, 1)
plt.scatter(X, Y, s=15, lw=.5, facecolor='none', label='Realized')
#plt.plot(U, doppler(U), lw=2, color='blue', label='True')
plt.fill_between(U, Yhat - 2*shat, Yhat + 2*shat, lw=0, color='red', alpha=.2, label='+2s')
plt.plot(U, Yhat, lw=2, color='red', label='Fitted')
plt.ylabel('Y')
plt.legend()
plt.xlim([0, 1])
ylim = plt.gca().get_ylim()
plt.title('Data')

plt.subplot(3, 1, 2)
plt.scatter(X, ehat, s=15, lw=.5, facecolor='none', label='Errors')
plt.axhline(color='black')
plt.ylim(ylim)
plt.xlim([0, 1])
plt.title('Errors')

plt.subplot(3, 1, 3)
plt.plot(U, sigma2hat**.5, lw=2, color='red', label='Estimate')
plt.plot(U, .1 * np.ones(U.shape), lw=2, color='blue', label='True')
plt.ylim([0, .4])
plt.xlim([0, 1])
plt.legend()
plt.xlabel('X')
plt.title('Conditional variance')

plt.tight_layout()
plt.show()
```

# Bandwidth selection

## Cross-validation criterion

$$\tilde{e}_{i}\left(h\right)=Y_{i}-\tilde{m}_{-i}\left(X_{i},h\right)$$

$$CV\left(h\right)=\frac{1}{n}\sum_{i=1}^{n}\tilde{e}_{i}\left(h\right)^{2}$$

$$\hat{h}=\arg\min_{h\geq h_{l}}CV\left(h\right)$$

```python
N = 500
X, Y, U = generate_data(N=N)

ktype = 'Gaussian'

H = np.linspace(.001, .05, 100)
CV = np.array([])
for h in H:
    ehat = error(Y, X, h, ktype)
    CV = np.append(CV, np.mean(ehat**2))

h = H[CV.argmin()]
Yhat = NW(U, X, Y, h=h, ktype=ktype)

ehat = error(Y, X, h, ktype)
sigma2hat = NW(U, X, ehat ** 2, h=h, ktype=ktype)
fxhat = ss.gaussian_kde(X)(U)
V2hat = roughness(ktype) * sigma2hat / fxhat / N / h
shat = V2hat**.5

plt.figure(figsize=(10, 5))
plt.plot(H, CV)
plt.scatter(h, CV.min(), facecolor='none', lw=2, s=100)
plt.xlim([H.min(), H.max()])
plt.xlabel('Bandwidth, h')
plt.ylabel('cross-validation, CV')
plt.show()
```

## Plot the (optimized) fit

```python
plt.figure(figsize=(10, 5))
#plt.plot(U, doppler(U), lw=2, color='blue', label='True')
plt.fill_between(U, Yhat - 2*shat, Yhat + 2*shat, lw=0, color='red', alpha=.2, label='+2s')
plt.plot(U, Yhat, lw=2, color='red', label='Fitted')
plt.scatter(X, Y, s=15, lw=.5, facecolor='none', label='Realized')
plt.xlim([0, 1])
plt.xlabel('X')
plt.ylabel('Y')
plt.legend()
plt.show()
```


# Complex plane

```{index} Complex plane
```

From the definition of complex numbers it is clear that there is a natural correspondence between a set of complex numbers $\mathbb{C}$ and points in a plane: to every complex number $z = x +iy$ we can *uniquely* assign a point $P(x, y)$ in the $\mathbb{R}^2$ plane. We call a plane in which every point has a corresponding complex number assigned to it a **complex plane** (or Argand plane). However, for simplicity we would normally simply say "in a point $z$" when we mean "in a point (of a complex plane) assigned a complex number $z$."

```python
z = complex(3, 2)

fig = plt.figure()
ax = fig.add_subplot(111)

plt.plot(z.real, z.imag, 'o', zorder=10)
plt.plot([3, 3], [0, 2], '--k', alpha=0.8)
plt.plot([0, 3], [2, 2], '--k', alpha=0.8)

ax.text(2.7, -0.25, "$Re(z)$", fontsize=12)
ax.text(-0.7, 1.95, "$Im(z)$", fontsize=12)

ax.arrow(-1., 0, 4.5, 0, shape='full', head_width=0.1, head_length=0.2)
ax.arrow(0, -1.5, 0, 4, shape='full', head_width=0.1, head_length=0.2)

ax.axis('equal')
ax.set_xlim(-2, 4)
ax.set_ylim(-2, 4)
ax.axis('off')

plt.show()
```

Having represented complex numbers geometrically, let us revisit the terms we introduced in the first notebook.

- Re $z$ is equal to the abscissa and Im $z$ to the ordinate of a point $P$. We therefore call x-axis the *real axis* and y-axis the *imaginary axis*.

- The modulus of a complex number $|z|$ is equal to the distance between the corresponding point $P$ and the origin of the complex plane. In general, the distance between points $P_1$ and $P_2$ assigned to numbers $z_1$ and $z_2$ is equal to the modulus of the difference of those two points:

$$ d(T_1, T_2) = \sqrt{(x_1 - x_2)^2 + (y_1 - y_2)^2} = | z_1 - z_2 |$$

- If a number $z$ is assigned a point $P(x, y)$, then the complex conjugate $z^*$ is assigned a point $P^*(x, -y)$, which is just a reflection of $P$ with respect to the real axis.

- Every point $P(x, y)$ has a corresponding vector $\vec{OP}$ from the origin to $P$. The sum $z_1 + z_2$ therefore represents vector addition \\( \overrightarrow{OP_1} + \overrightarrow{OP_2} \\).

Because of the vector analogy, we can also say that complex numbers satisfy the *triangle inequalities*:

$$ | z_1| - |z_2| \leq |z_1 + z_2| \leq |z_1| + |z_2| $$

(comp_trig_form)=
# Trigonometric form

```{sidebar} Complex plane - Argand diagram

Source: [Wikipedia](https://en.wikipedia.org/wiki/Complex_plane)
```

We can also observe the complex plane in polar coordinates $(r, \phi)$ which are related to the Cartesian coordinates $(x, y)$ through:

$$ x = r \cos \varphi, \quad y = r \sin \varphi $$

This leads to the *trigonometric form* of a complex number $z$:

$$ z = r(\cos \varphi + i \sin \varphi), $$

where $r$ is the **modulus** (or magnitude), equal to the absolute value of the complex number, a direct consequence of Pythagora's theorem:

$$ r = \sqrt{x^2 + y^2} = |z| $$

and the polar angle $\varphi$ is defined (to a multiply $2\pi$) by:

$$ \tan (\varphi + 2n \pi) = \frac{y}{x}, \quad n \in \mathbb{Z}, x \neq 0. $$

The angle $\varphi$ is called the **argument** (or phase) of a complex number $z$ and we write $\text{Arg}(z) = \varphi$. A special case is the complex number $z=0$ for which $r = 0$ and its argument is not defined.

Finding $\varphi$ is delicate because $\tan$ is a multivalued function. To avoid ambiguity, the simplest choice is $n = 0$ so that the interval is of length $2\pi$ and \\( - \pi < \text{arg}(z) \leq \pi \\). The value of $\text{Arg}(z)$ with $n=0$ is called the **principal value** of the argument. With this:

$$ \text{arg}(1) = 0, \quad \text{arg}(i) = \frac{\pi}{2}, \quad \text{arg}(-1) = \pi, \quad \text{arg}(-i) = -\frac{\pi}{2}, \quad \text{etc.} $$

THe relationship between $\text{Arg}(z)$ and $\text{arg}(z)$ is therefore:

$$ \text{Arg}(z) = \text{arg}(z) + 2n \pi, \quad n \in \mathbb{Z}.$$

Multiplying two complex numbers results in their absolute values being multiplied and the arguments being added:

```{margin} Angle addition formulae
$$ \sin(\alpha + \beta) = \sin \alpha \cos \beta + \cos \alpha \sin \beta \\
\cos (\alpha + \beta) = \cos \alpha \cos \beta - \sin \alpha \sin \beta$$
```

$$ \begin{align}
z_1 \cdot z_2 & = r_1(\cos \varphi_1 + i \sin \varphi_1) \cdot r_2(\cos \varphi_2 + i \sin \varphi_2) \\
& = r_1 r_2 (\cos (\varphi_1 + \varphi_2) + i \sin(\varphi_1 + \varphi_2))
\end{align} $$

For the multiplicative inverse we have

$$ \begin{align}
z^{-1} & = \frac{1}{r(\cos \varphi + i \sin \varphi)} \cdot \frac{cos \varphi - i \sin \varphi}{cos \varphi - i \sin \varphi} = \frac{\cos \varphi - i \sin \varphi}{r( \cos^2 \varphi + \sin^2 \varphi)} \\
& = \frac{1}{r} (\cos \varphi - i \sin \varphi) \end{align} $$

```{admonition} Properties of $|z|$ and arg($z$)
Let us summarise our findings in the form of the following equalities:

$$ | z_1 \cdot z_2 | = |z_1| \cdot |z_2| = r_1 r_2, \quad \text{arg}(z_1 \cdot z_2) = \text{arg}(z_1) + \text{arg}(z_2), $$

$$|z^{-1}| = |z|^{-1} = r^{-1}, \quad \text{arg}(z^{-1}) = -\text{arg}(z) $$

where $z_1, z_2 \neq 0$. By induction we get:

$$ |z^n| = |z|^n, \quad \text{arg}(z^n) = n\text{arg}(z), \quad \forall n \in \mathbb{Z}. $$
```

(comp_polar_form)=
# Polar form

```{index} Polar form of complex numbers
```

The property \\( \text{arg}(z_1 \cdot z_2) = \text{arg}(z_1) + \text{arg}(z_2) \\) might remind us of logarithms, where $\log (a \cdot b) = \log a + \log b$. This is not a coincidence!

The exponential function, which we will look at in the next chapter, allows us to write complex numbers in a *polar form*:

$$ z = r e^{i \varphi} $$

```{margin} [Euler's identity](https://en.wikipedia.org/wiki/Euler%27s_identity)
For a special angle $\varphi = \pi$ the Euler formula gives the Euler's identity:

$$ e^{i \pi} + 1 = 0 $$

It is considered to be one of the most beautiful equations in mathematics.
```

where we have used the **Euler's formula**:

$$ e^{i \varphi} = \cos(\varphi) + i \sin (\varphi). $$

In this representation, certain operations become much easier. For example,

$$ z_1 \cdot z_2 = r_1 r_2 e^{i(\varphi_1 + \varphi_2)}, \quad z^n = r^n e^{in \varphi} $$

```{index} De Moivre's formula
```

for all powers $n \in \mathbb{Z}$. If we write this using a complex number with unit modulus we recover **de Moivre's formula**:

$$ (\cos \varphi + i \sin \varphi)^n = (e^{i \varphi})^n = e^{in \varphi} = \cos (n \varphi) + i \sin (n \varphi). $$

Tip: Trigonometric identities

Trigonometric identities are also much easier to be recovered this way. For example, let us think about angle addition $\theta + \varphi$.

$$ e^{i(\theta + \varphi)} = e^{i \theta} e^{i \varphi} $$

We Apply Euler's formula to each complex number.

$$\begin{align}
\cos (\theta + \varphi) + i \sin(\theta + \varphi)
& = (\cos \theta + i \sin \theta)(\cos \varphi + i \sin \varphi) \\
& = \cos \theta \cos \varphi + i \cos \theta \sin \varphi + i \sin \theta \cos \varphi + i^2 \sin \theta \sin \varphi \\
& = (\cos \theta \cos \varphi - \sin \theta \sin \varphi) + i(\cos \theta \sin \varphi + \sin \theta \cos \varphi) \end{align} $$

Equate real and imaginary parts on both sides:

$$\cos (\theta + \varphi) = \cos \theta \cos \varphi - \sin \theta \sin \varphi \\
\sin (\theta + \varphi ) = \cos \theta \sin \varphi + \sin \theta \cos \varphi $$

The reader is encouraged to try to recover other trigonometric identities.


$$ \text{LaTeX command declarations here.}
\newcommand{\R}{\mathbb{R}}
\renewcommand{\vec}[1]{\mathbf{#1}}
$$

# EECS445 Machine Learning
## Discussion 03:  Linear Regression & Naive Bayes
Written by Zhao Fu; Edited by Chansoo Lee

### Linear Regression: Notations

- Let vector $\vec{x}_n \in \mathbb{R}^D$ denote the $n\text{th}$ data. $D$ denotes number of attributes in dataset.
- Let vector $\phi(\vec{x}_n) \in \mathbb{R}^M$ denote features for data $\vec{x}_n$. $\phi_j(\vec{x}_n)$ denotes the $j\text{th}$ feature for data $x_n$.
- Feature $\phi(\vec{x}_n)$ is the *artificial* features which represents the preprocessing step. $\phi(\vec{x}_n)$ is usually some combination of transformations of $\vec{x}_n$. For example, $\phi(\vec{x})$ could be vector constructed by $[\vec{x}_n^T, \cos(\vec{x}_n)^T, \exp(\vec{x}_n)^T]^T$. If we do nothing to $\vec{x}_n$, then $\phi(\vec{x}_n)=\vec{x}_n$.
- Continuous-valued label vector $t \in \mathbb{R}^N$ (target values). $t_n \in \mathbb{R}$ denotes the target value for $i\text{th}$ data.

### Recall: Least Squares Error Function
- We will find the solution $\vec{w}$ to linear regression by minimizing a cost/objective function.
- When the objective function is sum of squared errors (sum differences between target $t$ and prediction $y$ over entire training data), this approach is also called **least squares**.
- The objective function is
$$
\begin{aligned}
E(\vec{w})
&= \frac12 \sum_{n=1}^N (y(\vec{x}_n, \vec{w}) - t_n)^2 \\
&= \frac12 \sum_{n=1}^N \left( \sum_{j=0}^{M-1} w_j\phi_j(\vec{x}_n) - t_n \right)^2 \\
&= \frac12 \sum_{n=1}^N \left( \vec{w}^T \phi(\vec{x}_n) - t_n \right)^2 \\
\end{aligned}
$$

### Exercise 3.1
Consider a data set in which each data point $t_n$ is associated with a weighting factor $r_n > 0$, so that the sum-of-squares error function becomes
$$
E(\vec{w}; \vec{r}) = \frac12 \sum_{n=1}^N r_n\left(\vec{w}^T \phi(\vec{x}_n) - t_n \right)^2
$$

Find an expression for the solution $\vec{w}^*$ that minimizes this error function. Give two
alternative interpretations of the weighted sum-of-squares error function in terms of
(i) data dependent noise variance and (ii) replicated data points.

#### Solution (Matrix Calculus)

We can write the above error function as
$E(\vec{w};\vec{r}) = \frac{1}{2} \|S(\Phi \vec{w} - t)\|_2^2$
where $S$ is an $N$-by-$N$ diagonal matrix with entries $\sqrt{r_n}$. Note $S^\top S = S S^\top$ is a diagonal matrix ($N$-by-$N$) with entries $r_n$. We denote $R = S^\top S$. We also know that

Similarly to hands-on lecture 4, $\frac{1}{2}\|S(\Phi \vec{w} - t)\|_2^2 = \frac{1}{2}(\vec{w}^\top (S \Phi)^\top S^\top (S \Phi) \vec{w} - 2\vec{w}^\top \Phi^\top S^\top S t - t^\top t)$, and thus
$$\frac{\partial E}{\partial \vec{w}} = \Phi^\top R \Phi \vec{w} - \Phi^\top R t.$$

By setting it equal to 0, we have the closed form solution $\vec{w} = (\Phi^\top R \Phi)^{-1}\Phi^\top Rt$

### Model Selection: Cross Validation
Suppose we are using the following linear regression model to fit a dataset
$$h_\theta(x) = \sum_{i=0}^{k}{\theta_i \phi_i(x)}$$
where $\phi_i(x) = x^i$, and wish to decide if $k$ should be $0, 1, \dots,$ or $10$. How can we automatically select a model?

For the sake of concreteness, we assume we have some finite set of models $M = \{M_1,\dots, M_d\}$ that we’re trying to select among.
For instance, in our first example above, the model $M_i$ would be an $i$th order polynomial regression model.

### Model Selection: Hold-out Cross Validation

Suppose we have a training dataset $S$.

In hold-out cross validation(also called simple cross validation), we do the following:
1. Randomly split $S$ into $S_{train}$ (say, 70% of the data) and $S_{cv}$ (the remaining 30%). Here, $S_{cv}$ is called the hold-out cross validation set.
2. Train each model $M_i$ on $S_{train}$ only, to get some hypothesis.
3. Select and output the hypothesis that had the smallest error $\epsilon_{S_{cv}}$ on the hold out cross validation set. (Recall, $\epsilon_{S_{cv}}$ denotes the empirical error on the set of examples in Scv.)

### Model Selection: K-Fold Cross Validation

Here is a method, called k-fold cross validation, that holds out less data each time:
1. Randomly split $S$ into $k$ disjoint subsets of $m/k$ training examples each. Let’s call these subsets $S_1,\dots, S_k$.
2. For each model $M_i$, we evaluate it as follows:
For $j = 1,\dots, k$
Train the model $M_i$ on $S_1 \cup \cdots \cup S_{j−1} \cup S_{j+1} \cup \cdots S_k$ (i.e., train on all the data except $S_j$) to get some hypothesis $h_{ij}$. Test the hypothesis $h_{ij}$ on $S_j$, to get $\epsilon_{S_j}(h_{ij})$. The estimated generalization error of model $M_i$ is then calculated as the average of the $\epsilon_{S_j}(h_{ij})$, which is $ \frac{1}{k}\sum_{j}{\epsilon_{S_j}(h_{ij})}$.
3. Pick the model $M_i$ with the lowest estimated generalization error, and retrain that model on the entire training set $S$. The resulting hypothesis is then output as our final answer.

### Bayesian Linear Regression
Recall **Maximum Likelihood Estimator(MLE)** for least square regression:

Given $\{ \vec{x}_n, t_n \}_{n=1}^N$, we want to find $\vec{w}_{ML}$ that maximizes data likelihood function
$$
\vec{w}_{ML}
=\arg \max p(\vec{t}|\mathcal{X},\vec{w},\beta)
=\arg \max \prod_{n=1}^N \mathcal{N}(t_n|\vec{w}^T\phi(\vec{x}_n),\beta^{-1})
$$
and by derivation we have shown in lecture $\vec{w}_{ML}$ is equivalent to the least squares solution $\hat{\vec{w}} = \Phi^\dagger \vec{t}$.

### Bayesian Linear Regression
Recall **MAP Estimator** for least square regression:

$$
\begin{aligned}
\vec{w}_{MAP}
&= \arg \max p(\vec{w}|\vec{t}, \mathcal{X},\beta) & & (\text{Posteriori Probability})\\
&= \arg \max \frac{p(\vec{w}, \vec{t}, \mathcal{X},\beta)}{p(\mathcal{X}, t, \beta)} \\
&= \arg \max \frac{p(\vec{t}|\vec{w}, \mathcal{X},\beta) p(\vec{w}, \mathcal{X}, \beta)}{p(\mathcal{X}, t, \beta)} \\
&= \arg \max p(\vec{t}|\vec{w}, \mathcal{X},\beta) p(\vec{w}, \mathcal{X}, \beta) & & (p(X, t, \beta) \text{ is irrelevant to} \ \vec{w})\\
&= \arg \max p(\vec{t}|\vec{w}, \mathcal{X},\beta) p(\vec{w}) p(\mathcal{X}) p(\beta) & & (\text{Independence}) \\
&= \arg \max p(\vec{t}|\vec{w}, \mathcal{X},\beta) p(\vec{w}) & & (\text{Likelihood} \times \text{Prior})
\end{aligned}
$$
Here, we assume $\vec{w} \sim \mathcal{N}(\vec{0}, \alpha^{-1}I)$.

### Exercise 3.2:
Suppose that we have already observed $N$ data points, the posterior distribution over $\vec{w}$ can be regarded as the prior for the next observation. So we can predict the next data point $(\vec{x}_{N+1}, t_{N+1})$ by maximize $p(t_{N+1}|\vec{x}_{N+1}, \vec{X}, \alpha, \beta)$. Derive the full expression for the posterior of the new data point.

We can also predict the expectation value of $t_{N+1}$ given $\vec{x}_{N+1}$ by $\mathbb{E}[t_{N+1}|\vec{x}_{N+1}]$ using the posterior probability.

### Review: Naive Bayes

- The essence of Naive Bayes is the **conditionally independence assumption**
    $$
    P(\vec{x} | y = c) = \prod_{d=1}^D P(x_d | y=c)
    $$
    i.e., given the label, all features are independent.

- The **full generative** model of Naive Bayes is:
    $$
    \begin{align}
    y       &\sim \mathrm{Categorical}(\pi) \\
    x_d | y=c &\sim \mathrm{Categorical}(\theta_{cd}) \quad \forall\, d = 1,\dots,D
    \end{align}
    $$
    with parameters:
    - $P(y = c) = \pi_c$, $\forall c = 1,\dots,C $
    - $\sum_{c=1}^C \pi_c = 1$ and $\pi_c \geq 0, \forall c=1,\dots,C$
    - $P(x_d = m| y = c) = \theta_{cdm}$ for every $d = 1,\dots,D, m = 1, \dots, M, c = 1, \dots, C$
    - $\sum_{m=1}^M \theta_{cdm} = 1$

- Conditional indepence assumption
    - Conditional independence: for any $i \neq j$, $P(X_i | X_j, Y) = P(X_i | Y)$
    - The implication is: $P(X_1, \ldots, X_n | Y) = P(X_1 | X_2, \ldots, X_n, Y) P(X_2,\ldots, X_n | Y)$.
    - By the Bayes theorem,
$$P(Y| X_1, \ldots, X_n) = \frac{P(Y)}{P(X_1, \ldots, X_n)} P(X_1,\ldots,X_n | Y) = \frac{P(Y)\prod_i P(X_i | Y)}{P(X_1,\ldots, X_n)}$$
    - When computing the MAP estimate $P(Y | X_1,\ldots, X_n)$, we can simply compare the numerator.
- Parameter $\pi$ and $\theta$ are learned from training data.
    - $\hat{\pi}_c = \frac{N_c}{N}$
    - $\hat{\theta}_{cdm} = \frac{N_{cdm}}{N_c}$

### Common problems
1. What if not all the words appear in a category, in which case we have $N_{cdm} = 0$?
2. What if some attributes are continues variables, not discrete?


# Direct methods for solving linear systems (homework)

**Exercise 1.** Let us consider the linear system $A\mathbf{x} = \mathbf{b}$ where
$$
  A =
  \begin{bmatrix}
  \epsilon & 1 & 2\\
  1 & 3 & 1 \\
  2 & 1 & 3 \\
  \end{bmatrix}.
$$

1. Find the range of values of $\epsilon \in \mathbb{R}$ such that the matrix $A$ is symmetric and positive definite.
**Suggestion**: use the *Sylvester's criterion* which states that  a symmetric matrix $A \in \mathbb{R}^{n \times n}$ is positive definite if and only if all the main minors (The main minors of $A \in \mathbb{R}^{n \times n}$ are the determinants of the submatrices $A_p = (a_{i,j})_{1 \leq i, j \leq p}$, $p = 1, ..., n$). of $A$ are positive.
2. What factorization is more suitable for solving the linear system $A\mathbf{x}=\mathbf{b}$ for the case $\epsilon=0$? Motivate the answer.
3. Compute the Cholesky factorization $A = R^T R$ for the case $\epsilon = 2$.
4. Given $\mathbf{b} = (1,1,1)^T$, solve the linear system by using the Cholesky factorization computed at the previous point.

**Exercise 2.** Let us consider the following matrix $A \in \mathbb R^{3 \times 3}$ depending on the parameter $\epsilon \in \mathbb R$:
$$
A =
\begin{bmatrix}
1 & \epsilon & -1 \\
\epsilon & \frac{35}3 & 1 \\
-1 & \epsilon & 2 \\
\end{bmatrix}.
$$

1. Calculate the values of the parameter $\epsilon \in \mathbb R$ for which the matrix $A$ is invertible (non singular).

2. Calculate the Gauss factorization $LU$ of the matrix $A$ (when non singular) for a generic value of the parameter $\epsilon \in \mathbb R$.

3. Calculate the values of the parameter $\epsilon \in \mathbb R$ for which the Gauss factorization $LU$ of the matrix $A$  (when non singular) exists and is unique.

4. Set $\epsilon = \sqrt{\frac{35}3}$ and use the pivoting technique to calculate the Gauss factorization $LU$ of the matrix $A$.

5. For $\epsilon=1$, the matrix $A$ is symmetric and positive definite. Calculate the corresponding Cholesky factorization of the matrix $A$, i.e. the upper triangular matrix with positive elements on the diagonal, say $R$, for which $A = R^T R$.

# EXERCISE 1

```python
from numpy import *
from matplotlib.pyplot import *
from sympy import *
init_printing()

x=symbols("x")
A=Matrix([[x,1,2],[1,3,1],[2,1,3]])
det(A)
print(A)

results=zeros(5)
temp=A.berkowitz_minors()
temp
#x>11/8
```

```python
A[0,0]=0
A.LUdecomposition()
```

$$\left ( \left[\begin{matrix}1 & 0 & 0\\0 & 1 & 0\\2 & -5 & 1\end{matrix}\right], \quad \left[\begin{matrix}1 & 3 & 1\\0 & 1 & 2\\0 & 0 & 11\end{matrix}\right], \quad \left [ \left [ 0, \quad 1\right ]\right ]\right )$$

# EXERCISE 2

```python
x=symbols("x")
A=Matrix([[1,x,-1],[x,35/3,1],[-1,x,2]])
det(A)
solve(det(A))
#linsolve(det(A),0)
#invertible when 3x^2+2x-11.6666666!=0
```

```python

```


# Local approximation of an univariate quadratic expression using linear regression

### Learning goals:
- Mathematical statement of the linear regression using the simples data: univariate continuous function (single feature)
- Simple implementation using this closed form solution
- Definition of the usual metrics
- Move on to use existing libraries from Scipy or SKLearn to perform linear fitting and to perform an analysis of the model
- Increase the capacity of the model with powers of the features (polynomial)
- Understand iterative methods based on gradient descent, including the stochastic gradient descent

```python
import math
import numpy as np
from numpy import random
import matplotlib.pyplot as plt
from sklearn import metrics as skMetrics
from sklearn.linear_model import LinearRegression as skLinearRegression
import scipy as sy
import sympy
import pandas
```

## Generator Model

```python
def generateBatch(N, stochastic = False):
    #
    xMin = 0
    xMax = 0.5
    #
    b = 0.35
    std = 0.01
    #
    if stochastic:
        x = random.uniform(xMin, xMax, N)
    else:
        x = np.linspace(xMin, xMax, N)
    yClean = x**4 + (x-0.3)**3 + b
    y =  yClean + random.normal(0, std, N)
    return (x, y, yClean)
```

### Train data

```python
xs = sympy.Symbol('x')
sympy.expand(xs**4 + (xs-0.3)**3 + 0.35)
```

$\displaystyle x^{4} + x^{3} - 0.9 x^{2} + 0.27 x + 0.323$

```python
N = 100000
xTrain, yTrain, yClean = generateBatch(N)
plt.plot(xTrain, yTrain, xTrain, yClean);
plt.title('Training data');
```

### Test data

```python
xTest, yTest, yTestClean = generateBatch(N)
```

# Closed form / analiticaly

Linear, 1st degree approximation of y:
\begin{align}
y = x w + b
\end{align}

Given $N_{feature} = 1$:

- $x$ is a $N_{sample}$ vector
- $w$ is a scalar
- $y$ is a $N_{sample}$ vector
- $b$ is a scalar

Using mean square error (Euclidian norm), we are looking for $\hat{\theta} = \{\hat{b}, \hat{w}\}$ such that:

\begin{align}
\hat{\theta} \in {min}_{\theta} \lVert x w + b - y \rVert_2^2
\end{align}

\begin{align}
g(\theta) & = \lVert x w + b - y \rVert_2^2 \\
& = \sum_{i=0}^n \left(x_i^2 w^2 + y_i^2 + b^2 + 2x_iwb  - 2x_iwy_i - 2y_ib \right)\\
\end{align}

Lookup the minimum through the partial derivates:

\begin{cases}
\frac{\partial{g(\theta)}}{\partial b} = \sum_{i=0}^n 2(x_iw + b - y_i) \\
\frac{\partial{g(\theta)}}{\partial w} = \sum_{i=0}^n 2(x_iw + b - y_i)x_i
\end{cases}

Let's define for any variable z:
\begin{align}
\overline{z_n} & = \frac 1n \sum_{i=0}^n z_i
\end{align}

Then:
\begin{align}
& \begin{cases}
\frac{\partial{g(\theta)}}{\partial b} = 0 \\
\frac{\partial{g(\theta)}}{\partial w} = 0
\end{cases}
\\ \iff &
\begin{cases}
\overline{x_n} w + b = \overline{y_n} & Eq1\\
\overline{x_n^2 }w + \overline{x_n} b = \overline{x_n y_n} & Eq2\\
\end{cases}
\end{align}

$Eq2 - Eq1 \overline{x_n}$ :

\begin{align}
& w \left( \overline{x_n^2} - \overline{x_n}^2 \right) = \overline{x_n y_n} - \overline{x_n}.\overline{y_n}\\
\end{align}

Leading to:

\begin{align}
w &= \frac{\overline{x_n y_n} - \overline{x_n}.\overline{y_n}}{\overline{x_n^2} - \overline{x_n}^2}\\
  &= \frac{\sum_{i=0}^n \left(x_i - \overline{x_n} \right) \left(y_i - \overline{y_n} \right)}{\sum_{i=0}^n \left(x_i - \overline{x_n} \right)  } \\
b &= \sum_{i=0}^n y_i - x_i w = \overline{y} - \overline{x} w  \\
\end{align}

Remove biases

```python
xUnB = xTrain - np.mean(xTrain)
yUnB = yTrain - np.mean(yTrain)
```

```python
# In case x is univariate, the matrix product x^T x is a scalar
wEst = np.dot(xUnB, yUnB) / np.dot(xUnB, xUnB)
bEst = np.mean(yTrain - wEst * xTrain)
print('Linear regression estimate: y = {:.3f} x + {:.3f}'.format(wEst, bEst))
```

    Linear regression estimate: y = 0.145 x + 0.323

### Test model

```python
yEst1 = wEst * xTest + bEst
```

```python
plt.plot(xTest, yTest, xTest, yEst1);
res = yTest - yEst1
mse1 = np.dot(res, res) / N
print('Closed form, MSE = {:.3e}'.format(mse1));
```

## Numpy polyfit, 1st degree
http://www.python-simple.com/python-numpy-scipy/fitting-regression.php

```python
fit2 = np.polyfit(xTrain, yTrain, 1)
fit2
```

    array([0.14488612, 0.32303991])

### Test model

```python
yEst2 = fit2[0] * xTest + fit2[1]
```

```python
plt.plot(xTest, yTest, xTest, yEst2);
mse2 = skMetrics.mean_squared_error(yTest, yEst2)
print('Numpy polyfit 1st degree, MSE =', "{:.3e}".format(mse2));
```

## Numpy polyfit, 4th degree

```python
fit3 = np.polyfit(xTrain, yTrain, 4)
fit3
```

    array([ 1.03565332,  0.96438978, -0.88586569,  0.26726443,  0.3231845 ])

### Test model

```python
yEst3 = xTest**4 * fit3[0] + xTest**3 * fit3[1] + xTest**2 * fit3[2] + xTest * fit3[3] + fit3[4]
```

```python
plt.plot(xTest, yTestClean, xTest, yEst3);
mse3 = skMetrics.mean_squared_error(yTest, yEst3)
plt.legend(['ori','fitted'])
print('Numpy polyfit 4th degre, MSE =', "{:.3e}".format(mse3));
```

## NumPy least square
http://www.python-simple.com/python-numpy-scipy/fitting-regression.php

```python
fit4, residues, rank, s = np.linalg.lstsq(np.reshape(xUnB, (N,1)), yUnB)
fit4
```

    //anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:1: FutureWarning: `rcond` parameter will change to the default of machine precision times ``max(M, N)`` where M and N are the input matrix dimensions.
    To use the future default and silence this warning we advise to pass `rcond=None`, to keep using the old, explicitly pass `rcond=-1`.
      """Entry point for launching an IPython kernel.

    array([0.14488612])

### Test model

```python
yEst4 = fit4 * xTest + bEst
```

```python
plt.plot(xTest, yTest, xTest, yEst4);
mse4 = skMetrics.mean_squared_error(yTest, yEst4)
print('Numpy Least square, MSE =', "{:.3e}".format(mse4));
```

## Scipy linear regression

http://www.python-simple.com/python-numpy-scipy/fitting-regression.php

```python
fit5 = sy.stats.linregress(xTrain, yTrain)
fit5
```

    LinregressResult(slope=0.14488612244983226, intercept=0.3230399081789981, rvalue=0.8633622269474659, pvalue=0.0, stderr=0.0002677762899562608)

### Test model

```python
yEst5 = fit5.slope * xTest + fit5.intercept
```

```python
plt.plot(xTest, yTest, xTest, yEst5)
mse5 = skMetrics.mean_squared_error(yTest, yEst5)
print('Scipy, MSE =', "{:.3e}".format(mse5));
```

## SK Learn linear regression

```python
model6 = skLinearRegression(normalize=False)
model6.fit(xTrain.reshape(-1,1), yTrain.reshape(-1,1))
print('SciKit Learn linear regression, b =', model6.intercept_[0], ', w =', model6.coef_[0][0])
```

    SciKit Learn linear regression, b = 0.323039908178998 , w = 0.1448861224498326

```python
print('SciKit Learn, R^2-score =', model6.score(xTest.reshape(-1,1), yTest))
```

    SciKit Learn, R^2-score = 0.7448545267081639

### Test model

```python
yEst6 = model6.predict(xTest.reshape(-1,1))
```

```python
plt.plot(xTest, yTest, xTest, yEst6)
mse6 = skMetrics.mean_squared_error(yTest, yEst6)
print('SciKit Learn, MSE =', "{:.3e}".format(mse6));
```

# Gradient descent

Let's first plot MSE for several values of the slope in order to verify the convex shape of the cost function

```python
Ns = 100
slope = np.linspace(-1, 1, Ns)
sx = np.matmul(np.reshape(xUnB, (N, 1)), np.reshape(slope,(1,Ns)))
er_sx_y = sx - np.reshape(yUnB, (N, 1))
mse = np.mean(er_sx_y.T**2, axis = 1)
```

```python
plt.plot(slope, mse)
plt.xlabel('w')
plt.ylabel('MSE')
plt.title('Min MSE = %.3e' % np.min(mse))
plt.grid();
```

Convexity of the MSE as function of the linear regression slope (w coefficient) is shown

### Gradient descent computation

```python
def calcGradient(X, Y, b, w):
    A = w * X + b - Y
    F_db = np.sum(A)
    F_dw = np.dot(A, X)
    return (F_db, F_dw, math.sqrt(F_db**2 + F_dw**2))
```

```python
# Initial coef
b7, w7 = 1, 1
threshold = 1e-1
learningRate = 1e-6
nIterMax = 1e5

# Init
gradient_b, gradient_w, gradientNorm = calcGradient(xTrain, yTrain, b7, w7)

print('START: b = %.3e' % b7, ', w = %.3e' % w7, ', Gradient norm = %.3e' % gradientNorm)
w7Learn = [np.array([b7, w7, gradientNorm])]

nIter = 0
while (gradientNorm > threshold) & (nIter < nIterMax):
    b7 = b7 - learningRate * gradient_b
    w7 = w7 - learningRate * gradient_w
    gradient_b, gradient_w, gradientNorm = calcGradient(xTrain, yTrain, b7, w7)
    w7Learn.append(np.array([b7, w7,  gradientNorm]))
    nIter += 1

print('END  : b = %.3e' % b7, ', w = %.3e' % w7, ', Gradient norm = %.3e' % gradientNorm, ', num iteration =', len(w7Learn))
df7 = pandas.DataFrame(w7Learn, columns = ('b', 'w', 'Gradient norm'))
```

    START: b = 1.000e+00 , w = 1.000e+00 , Gradient norm = 9.226e+04
    END  : b = 3.230e-01 , w = 1.449e-01 , Gradient norm = 9.981e-02 , num iteration = 4832

```python
fig = plt.figure(figsize=(16,12))
plt.subplot(2,2,1)
plt.plot(df7['b'])
plt.grid()
plt.title('b');
plt.subplot(2,2,2)
plt.plot(df7['w'])
plt.grid()
plt.title('w');
plt.subplot(2,2,3)
plt.semilogy(df7['Gradient norm'])
plt.grid()
plt.title('Gradient norm');
```

```python
fig = plt.figure(figsize=(16,8))
plt.subplot(1,2,1)
plt.semilogy(df7['b'], df7['Gradient norm'])
plt.grid()
plt.xlabel('b')
plt.ylabel('gradient norm');
plt.title('Trajectory of w and Gradient norm');
plt.subplot(1,2,2)
plt.semilogy(df7['w'], df7['Gradient norm'])
plt.grid()
plt.xlabel('w')
plt.ylabel('gradient norm');
plt.title('Trajectory of w and Gradient norm');
```

### Test model

```python
yEst7 = w7*xTest + b7
```

```python
plt.plot(xTest, yTest, xTest, yEst7);
mse7 = skMetrics.mean_squared_error(yTest, yEst7)
print('Gradient descent, MSE =', '{:.3e}'.format(mse7));
```

## Stochastic gradient descent

Using new data batch at each iteration.

Alternatively, on a finite data set: shuffle the samples.

```python
nBatch = 100
b8, w8 = 2, 2
threshold = 1e-3
learningRate = 1e-3
nIterMax = 1e5

# Initial batch
xBatch, yBatch, yBC = generateBatch(nBatch, True)
gradient_b, gradient_w, gradientNorm = calcGradient(xBatch, yBatch, b8, w8)

print('START: b = %.3e' % b8, ', w = %.3e' % w8, ', Gradient norm = %.3e' % gradientNorm)
w8Learn = [np.array([b8, w8, gradientNorm])]

# Continue
nIter = 0
while (gradientNorm > threshold) & (nIter < nIterMax):
    b8 = b8 - learningRate * gradient_b
    w8 = w8 - learningRate * gradient_w
    xBatch, yBatch, yBC = generateBatch(nBatch, True)
    gradient_b, gradient_w, gradientNorm = calcGradient(xBatch, yBatch, b8, w8)
    w8Learn.append(np.array([b8, w8,  gradientNorm]))
    learningRate = learningRate * 0.9999 # Decreasing learning rate
    nIter += 1

print('START: b = %.3e' % b8, ', w = %.3e' % w8, ', Gradient norm = %.3e' % gradientNorm, ', num iteration =', len(w8Learn))
df8 = pandas.DataFrame(w8Learn, columns = ('b', 'w', 'Gradient norm'))
```

    START: b = 2.000e+00 , w = 2.000e+00 , Gradient norm = 2.230e+02
    START: b = 3.219e-01 , w = 1.492e-01 , Gradient norm = 9.648e-04 , num iteration = 3495

```python
fig = plt.figure(figsize=(16,12))
plt.subplot(2,2,1)
plt.plot(df8['b'])
plt.grid()
plt.title('b');
plt.subplot(2,2,2)
plt.plot(df8['w'])
plt.grid()
plt.title('w');
plt.subplot(2,2,3)
plt.semilogy(df8['Gradient norm'])
plt.grid()
plt.title('Gradient norm');
```

The gradient norm is getting very noisy as the value is below $10^{-1}, a more adaptive learning rate would be needed there and a mean on the gradient norm to improve the stop condition of the gradient descent

```python
fig = plt.figure(figsize=(16,8))
plt.subplot(1,2,1)
plt.semilogy(df8['b'], df8['Gradient norm'])
plt.grid()
plt.xlabel('b')
plt.ylabel('gradient norm');
plt.title('Trajectory of w and Gradient norm');
plt.subplot(1,2,2)
plt.semilogy(df8['w'], df8['Gradient norm'])
plt.grid()
plt.xlabel('w')
plt.ylabel('gradient norm');
plt.title('Trajectory of w and Gradient norm');
```

### Test model

```python
yEst8 = w8*xTest + b8
```

```python
plt.scatter(xBatch, yBatch, marker='.', color = 'black');
plt.plot(xTrain, yClean)
plt.plot(xTest, yEst8)
plt.legend(['Generator model', 'Estimated model', 'Last batch'])
mse8 = skMetrics.mean_squared_error(yTest, yEst8)
print('Stochastic gradient descent, MSE =', "{:.3e}".format(mse8));
```

## Gradient descent with SciKit Learn

https://scikit-learn.org/stable/modules/sgd.html#regression

```python
from sklearn.linear_model import SGDRegressor as skSGDRegressor

model9 = skSGDRegressor(alpha=0.0001, average=False,
           early_stopping=True, epsilon=0.1, eta0=0.0, fit_intercept=True,
           learning_rate='optimal', loss='squared_loss', max_iter=1000,
           n_iter_no_change=5, penalty='l2', power_t=0.5,
           random_state=None, shuffle=True, tol=0.001,
           validation_fraction=0.1, verbose=0, warm_start=False)
# l1_ratio=0.15,
```

Notes:
- Regularizer is called 'penalty' and parameterized by 'alpha' (and 'l1_ratio')
- Early stopping is available and parameterized by 'early_stopping', 'max_iter', 'tol' and 'n_iter_no_change'
- Shuffling between epochs enabled by 'shuffle'

```python
model9.fit(xTrain.reshape(-1, 1), yTrain)
print('Y = {0} X + {1}'.format(model9.coef_, model9.intercept_))
```

    Y = [0.14393348] X + [0.32146107]

```python
yEst9 = model9.predict(xTest.reshape(-1,1))
```

```python
plt.plot(xTrain, yClean)
plt.plot(xTest, yEst9)
plt.legend(['Generator model', 'Estimated model'])
mse9 = skMetrics.mean_squared_error(yTest, yEst9)
print('Gradient descent with SK Learn, MSE =', "{:.3e}".format(mse9));
```

# Main take-aways

Linear regression has a closed form leading to the best fit. Many Python libraries provide this linear or polynomial fitting.

We have also learnt gradient descent on this simple case, it will be very useful for coming projects based on neural networks.

# Where to go from here ?

__Other single feature linear implementation__ [using TensorFlow](LinearRegressionUnivariate-TensorFlow.html) ([Notebook](LinearRegressionUnivariate-TensorFlow.ipynb))

__More complex bivariate models__ [using "raw" Python](LinearRegressionBivariate.html) ([Notebook](LinearRegressionBivariate.ipynb)) up to the gradient descent with regularizer, or [using Keras](LinearRegressionBivariate-Keras.html) ([Notebook](LinearRegressionBivariate-Keras.ipynb))

__Compare with the single feature binary classification using logistic regression__ [using "raw" Python or libraries](../classification/ClassificationContinuousSingleFeature.html) ([Notebook](../classification/ClassificationContinuousSingleFeature.ipynb]))


```python
import numpy as np
import matplotlib.pyplot as plt
# g/kg
xmax = 30.0
xmin = 0.0
dx = 0.01

x = np.arange(xmin+dx, xmax+dx, dx)

A = 0.5 * ( xmax + xmin )
B = 0.5 * ( xmax - xmin )
```

Kotsuki et al. (2018)
$$
x=1/2(x_{\rm max} + x_{\rm min}) + 1/2(x_{\rm max}-x_{\rm min})\tanh(y)\\
x=A+B\tanh(y)
$$
where $x$ is a control variable in physical scape, $y$ is a transformed control variable, $x_{\rm max}$ and $x_{\rm min}$ are the maximum and minimum values for x.

Inversely,
$$
\tanh(y) = (2x-(x_{\rm max} + x_{\rm min}))/(x_{\rm max}-x_{\rm min})\\
y = 1/2\ln \frac{(1+z)}{(1-z)}
$$
where $z=(2x-(x_{\rm max} + x_{\rm min}))/(x_{\rm max}-x_{\rm min})$, or $z=\frac{x-A}{B}$

```python
def y2x_k( x ):
    return( A+B*np.tanh(x) )

def x2z( x ):
    return( ( x-A ) / B )

def z2x( z ):
    return( (z*B+A) )

def x2y_k( x ):
    z = x2z( x )
    return( 0.5*np.log( (1+z)/(1-z) ) )

```

```python
y_k = x2y_k(x)
_ = plt.plot( x, y_k)
plt.xlabel( "control variable (x)")
plt.ylabel( "transformed control variable (y)")
```

What if use the above transformation for analysis?
Analysis equation is:
$$
y_a = y_b + \kappa (y_o - y_b).
$$
This can be rewritten by $z(x)$:
$$
\begin{align}
1/2\ln \frac{(1+z_a)}{(1-z_a)} &= 1/2\ln \frac{(1+z_b)}{(1-z_b)} +\kappa \left( 1/2\ln \frac{(1+z_o)}{(1-z_o)} - 1/2\ln \frac{(1+z_b)}{(1-z_b)}\right)\\
\ln \frac{(1+z_a)}{(1-z_a)} &= \ln \frac{(1+z_b)}{(1-z_b)} +\kappa \left( \ln \frac{(1+z_o)(1-z_b)}{(1-z_o)(1+z_b)} \right)\\
\ln \frac{(1+z_a)}{(1-z_a)} &= \ln \frac{(1+z_o)^{\kappa}(1-z_b)^{\kappa-1}}{(1-z_o)^{\kappa}(1+z_b)^{\kappa-1}}\\
\frac{(1+z_a)}{(1-z_a)} &= \frac{(1+z_o)^{\kappa}(1-z_b)^{\kappa-1}}{(1-z_o)^{\kappa}(1+z_b)^{\kappa-1}}\\
\left( 1+ \frac{(1+z_o)^{\kappa}(1-z_b)^{\kappa-1}}{(1-z_o)^{\kappa}(1+z_b)^{\kappa-1}}\right)z_a &= \frac{(1+z_o)^{\kappa}(1-z_b)^{\kappa-1}}{(1-z_o)^{\kappa}(1+z_b)^{\kappa-1}}-1\\
\left( (1-z_o)^{\kappa}(1+z_b)^{\kappa-1}+ (1+z_o)^{\kappa}(1-z_b)^{\kappa-1}\right)z_a &= (1+z_o)^{\kappa}(1-z_b)^{\kappa-1}-(1-z_o)^{\kappa}(1+z_b)^{\kappa-1}\\
z_a &= \frac{ (1+z_o)^{\kappa}(1-z_b)^{\kappa-1}-(1-z_o)^{\kappa}(1+z_b)^{\kappa-1} }{(1-z_o)^{\kappa}(1+z_b)^{\kappa-1}+ (1+z_o)^{\kappa}(1-z_b)^{\kappa-1}}\\
\end{align}
$$

```python
def get_za( zb, zo, kappa ):
    tmp = np.power( 1+zo, kappa) * np.power( 1-zb, kappa-1) - np.power( 1-zo, kappa )* np.power(1+zb, kappa-1)
    return( tmp / (np.power( 1-zo, kappa) * np.power( 1+zb, kappa-1) + np.power( 1+zo, kappa )* np.power(1-zb, kappa-1)
  ) )
```

```python
xb = 0.001
xo = 0.99

kappa_l = np.arange( 0.0, 1.05, 0.05 )

xa_l_not = xb + kappa_l*( xo - xb)

xa_log_l = np.log( xb ) + kappa_l*( np.log( xo ) - np.log( xb ) )
xa_log_l = np.exp( xa_log_l)

zb = x2z( xb )
zo = x2z( xo )

za_l = get_za( zb, zo, kappa_l)
xa_l = z2x( za_l )

_ = plt.plot( kappa_l, xa_l, label="w/ transformation", color='r')
_ = plt.plot( kappa_l, xa_l_not, label="No transformation", color='k')
_ = plt.plot( kappa_l, xa_log_l, label="w/ log transformation", color='b')

_ = plt.ylabel( "Analyzed variable (x)")
_ = plt.xlabel( "Kalman gain")
_ = plt.grid('on')
_ = plt.legend( loc='upper left')
_ = plt.suptitle( "" )
```

```python

```


```python
import sympy as sp
sp.init_printing()
```

```python
a,Δt,x = sp.symbols('a,Δt,x')
```

Deriválás

```python
sp.diff(sp.sin(x**3),x)
```

Többszöri deriválás

```python
sp.diff(sp.sin(x**3),x,3)
```

Integrálás

```python
sp.integrate(1/(1+x),x)
```

Határozott integrál

```python
sp.integrate(1/(1+x),(x,1,2))
```

```python
a = sp.Symbol('a')
```

```python
sp.integrate(1/(x**2 + a),x)
```

Láncszabályt is ismeri a szoftver

```python
y = sp.Symbol('y')
def f(x):
    return x**2
def g(y):
    return sp.sin(y)
```

```python
f(g(y))
```

```python
sp.diff(f(g(y)),y)
```

```python
sp.diff(g(f(x)),x)
```

Sok esetben nem létezik zárt alakú kifejezés a határozatlan integrálhoz. Ebben az esetben használhatjuk a [numerikus integrálást](https://en.wikipedia.org/wiki/Numerical_integration) a határozott integrál számításához:

```python
sp.integrate(sp.sin(sp.cos(x)),x)
```

[numerikus intágráls SymPy-vel](https://docs.sympy.org/latest/modules/integrals/integrals.html#numeric-integrals)

nem triviális


# Lecture 7: Gambler's Ruin & Random Variables

## Gambler's Ruin

2 gamblers $A$ and $B$ are successively playing a game until one wins all the money and the other is ruined (goes bankrupt). There is a sequence of rounds, with a one dollar bet each time. The rounds are independent events. Let $p = P(\text{A wins a certain round})$ and the inverse is $q = 1 - p$, by convention.

_What is the probability that $A$ wins the entire game?_

Some clarifications:

* there is a total of $N$ dollars in this closed system game (no other money comes into play)
* $A$ starts with $i$ dollars, $B$ starts with $N-i$ dollars

But where do we begin to solve this problem?

### Random Walk

A [random walk](https://en.wikipedia.org/wiki/Random_walk) between 2 points on a number line is very similar to the Gambler's Ruin.

How many rounds could a game last? Is it possible for the game to continue on to infinity?

Well, notice how this has a very nice __recursive nature__. If $A$ loses a round, the game can be seen as starting anew at $i-1$, and if he wins, the game would start anew at $i+1$. It is the same problem, but with a different starting condition.

### Strategy

Conditioning on the _first step_ is called __first step analysis__.

Let $P_i = P(\text{A wins the entire game|A starts with i dollars})$. Then from the Law of Total Probability, we have:

\begin{align}
    P_i &= p P_{i+1} + q P_{i-1} \text{, } & &\text{where }1 \lt i \lt N-1 \\
    & & & P_0 = 0 \\
    & & & P_N = 1 \\
\end{align}

See how this is a recursive equation? This is called a [__difference equation__](http://mathworld.wolfram.com/DifferenceEquation.html), which is a discrete analog of a differential equation.

### Solving the Difference Equation

\begin{align}
    P_i &= p P_{i+1} + q P_{i-1} & & \\
    \\
    \\
    P_i &= x^i & &\text{see what happens when we guess with a power} \\
    \Rightarrow x^i &= p x^{i+1} + q x^{i-1} \\
    0 &= p x^2 - x + q & &\text{factoring out } x^{i-1} \text{, we are left with a quadratic}\\
    \\
    x &= \frac{1 \pm \sqrt{1-4pq}}{2p} & &\text{solving with the quadratic formula} \\
      &= \frac{1 \pm \sqrt{(2p-1)^2}}{2p} & &\text{since }1-4pq = 1-4p(1-p) = 4p^2 - 4p - 1 = (2p -1)^2 \\
      &= \frac{1 \pm (2p-1)}{2p} \\
      &\in \left\{1, \frac{q}{p} \right\} \\
    \\
    \\
    P_i &= A(1)^i + B\left(\frac{q}{p}\right)^i & &\text{if } p \neq q~~~~ \text{(general solution for difference equation)} \\
    \Rightarrow B &= -A & &\text{from }P_0 = 1\\
    \Rightarrow 1 &= A(1)^N + B\left(\frac{q}{p}\right)^N & &\text{from } P_N = 1\\
    &= A(1)^N - A\left(\frac{q}{p}\right)^N \\
    &= A\left(1-\frac{q}{p}\right)^N \\
    \\
    \\
    \therefore P_i &=
        \begin{cases}
            \frac{1-\left(\frac{q}{p}\right)^i}{1-\left(\frac{q}{p}\right)^N}  & \quad \text{ if } p \neq q \\
            \frac{i}{N}  & \quad \text{ if } p = q \\
        \end{cases}
\end{align}

### Example calculations of $P_i$ over a range of $N$

Assuming an unfair game:

```python
import math

def gamblers_ruin(i, p, q, N):
    if math.isclose(p,q):
        return i/N
    else:
        return ((1 - (q/p)**i)) / (1 - (q/p)**N)

p = 0.49
q = 1.0 - p

N = 20
i = N/2
print("With N={} and p={}, probability that A wins all is {:.2f}".format(N, p, gamblers_ruin(i, p, q, N)))

N = 100
i = N/2
print("With N={} and p={}, probability that A wins all is {:.2f}".format(N, p, gamblers_ruin(i, p, q, N)))

N = 200
i = N/2
print("With N={} and p={}, probability that A wins all is {:.2f}".format(N, p, gamblers_ruin(i, p, q, N)))
```

    With N=20 and p=0.49, probability that A wins all is 0.40
    With N=100 and p=0.49, probability that A wins all is 0.12
    With N=200 and p=0.49, probability that A wins all is 0.02

And assuming a fair game:

```python
p = 0.5
q = 1.0 - p

N = 20
i = N/2
print("With N={} and p={}, probability that A wins all is {:.2f}".format(N, p, gamblers_ruin(i, p, q, N)))

N = 100
i = N/2
print("With N={} and p={}, probability that A wins all is {:.2f}".format(N, p, gamblers_ruin(i, p, q, N)))

N = 200
i = N/2
print("With N={} and p={}, probability that A wins all is {:.2f}".format(N, p, gamblers_ruin(i, p, q, N)))
```

    With N=20 and p=0.5, probability that A wins all is 0.50
    With N=100 and p=0.5, probability that A wins all is 0.50
    With N=200 and p=0.5, probability that A wins all is 0.50

#### Could the game ever continue forever on to infinity?

No, but you cannot really see so that would indicate such at first.

Recall that we have the following solution to the difference equation for the Gambler's Ruin game:

\begin{align}
    P_i &=
        \begin{cases}
            \frac{1-\left(\frac{q}{p}\right)^i}{1-\left(\frac{q}{p}\right)^N}  & \quad \text{ if } p \neq q \\
            \frac{i}{N}  & \quad \text{ if } p = q \\
        \end{cases}
\end{align}

The only time you'd think the game could continue on to infinity is when $p=q$. But

\begin{align}
    P(\Omega) &= 1\\
    &= P(\text{A wins all}) + P(\text{B wins all}) \\
    &= P_i + P_{N-i} \\
    &= \frac{i}{N} + \frac{N-i}{N}
\end{align}

The above implies that aside from the case where $A$ wins all, and the case where $B$ wins all, there is no other event in $\Omega$ to consider, hence the game can never continue on to infinity without either side winning.

This also means that unless $p=q$, you are __will__ lose your money, and the only question is how fast will you lose it.

# Random Variables

Consider these statements:

\begin{align}
    x + 2 &= 9 \\
    x &= 7
\end{align}

_What is a variable?_
* variable $x$ is a symbol that we use as a substitute for an arbitrary _constant_ value.

_What is a __random__ variable?_

* This is not a _variable_, but a __function from the sample space $S$ to $\mathbb{R}$__.
* It is a "summary" of an aspect of the experiment (this is where the randomness comes from)

Here are a few of the most useful _discrete random variables_.

## Bernoulli Distribution

### Description
A probability distribution of a random variable that takes the value 1 in the case of a success with probability $p$; or takes the value 0 in case of a failure with probability $1-p$.

A most common example would be a coin toss, where heads might be considered a success with probability $p=0.5$ if the coin is a fair.

A random variable $x$ has the Bernoulli distribution if
- $x \in \{0, 1\}$
- $P(x=1) = p$
- $P(x=0) = 1-p$

### Notation

$X \sim \text{Bern}(p)$

### Parameters

$0 < p < 1 \text{, } p \in \mathbb{R}$

### Probability mass function

The probability mass function $P(x)$ over possible values $x$

\begin{align}
  P(x) =
  \begin{cases}
    1-p, &\text{ if } x = 0 \\
    p, &\text{ if } x = 1 \\
  \end{cases} \\
\end{align}

### Expected value

\begin{align}
  \mathbb{E}(X) &= 1 P(X=1) + 0 P(X=0) \\
                &= p
\end{align}

### Special case: Indicator random variables (r.v.)

\begin{align}
  &X =
  \begin{cases}
    1, &\text{ if event A occurs} \\
    0, &\text{ otherwise} \\
  \end{cases} \\
  \\
  \\
  \Rightarrow &\mathbb{E}(X) = P(A)
\end{align}

## Binomial Distribution

### Description

The distribution of the number of successes in $n$ independent Bernoulli trials $Bern(p)$, where the chance of success $p$ is the same for all trials $n$.

Another case might be a string of indicator random variables.

### Notation

$X \sim \text{Bin}(n, p)$

### Parameters

- $n \in \mathbb{N}$
- $p \in [0,1]$

### Probability mass function

\begin{align}
    P(x=k) &= \binom{n}{k} p^k (1-p)^{n-k}
\end{align}

### Expected value

\begin{align}
    \mathbb{E}(X) = np
\end{align}

## In parting...

Now think about this true statement as we move on to Lecture 3:

\begin{align}
    & X \sim \text{Bin}(n,p) \text{, } Y \sim \text{Bin}(m,p) \\
    & \rightarrow X+Y \sim \text{Bin}(n+m, p)
\end{align}

----

## Appendix A: Solving $P_i$ when $p=q$ using l'Hopital's Rule

To solve for for the case where $p = q$, let $x = \frac{q}{p}$.

\begin{align}
    lim_{x \rightarrow 1}{\frac{1-x^i}{1-x^N}} &= lim_{x\rightarrow1}{\frac{ix^{i-1}}{Nx^{N-1}}} &\text{ by l'Hopital's Rule} \\
    &= \frac{i}{N}
\end{align}


# Rank-one nonnegative matrix factorization

The DGP atom library has several functions of positive matrices, including the trace, (matrix) product, sum, Perron-Frobenius eigenvalue, and $(I - X)^{-1}$ (eye-minus-inverse). In this notebook, we use some of these atoms to approximate a partially known elementwise positive matrix
as the outer product of two positive vectors.

We would like to approximate $A$ as the outer product of two positive vectors $x$ and $y$, with $x$ normalized so that the product of its entries equals $1$. Our criterion is the average relative deviation between the entries of $A$ and
$xy^T$, that is,

$$
\frac{1}{mn} \sum_{i=1}^{m} \sum_{j=1}^{n} R(A_{ij}, x_iy_j),
$$

where $R$ is the relative deviation of two positive numbers, defined as

$$
R(a, b) = \max\{a/b, b/a\} - 1.
$$

The corresponding optimization problem is

$$
\begin{equation}
\begin{array}{ll}
\mbox{minimize} & \frac{1}{mn} \sum_{i=1}^{m} \sum_{j=1}^{n} R(X_{ij}, x_iy_j)
\\
\mbox{subject to} & x_1x_2 \cdots x_m = 1 \\
& X_{ij} = A_{ij}, \quad \text{for } (i, j) \in \Omega,
\end{array}
\end{equation}
$$

with variables $X \in \mathbf{R}^{m \times n}_{++}$, $x \in \mathbf{R}^{m}_{++}$, and $y \in \mathbf{R}^{n}_{++}$. We can cast this problem as an equivalent generalized geometric program by discarding the $-1$ from the relative deviations.

The below code constructs and solves this optimization problem, with specific problem data

$$
A = \begin{bmatrix}
1.0 & ? &  1.9 \\
? & 0.8 &  ? \\
3.2 & 5.9&  ?
\end{bmatrix},
$$

```python
import cvxpy as cp

m = 3
n = 3
X = cp.Variable((m, n), pos=True)
x = cp.Variable((m,), pos=True)
y = cp.Variable((n,), pos=True)

outer_product = cp.vstack([x[i] * y for i in range(m)])
relative_deviations = cp.maximum(
  cp.multiply(X, outer_product ** -1),
  cp.multiply(X ** -1, outer_product))
objective = cp.sum(relative_deviations)
constraints = [
  X[0, 0] == 1.0,
  X[0, 2] == 1.9,
  X[1, 1] == 0.8,
  X[2, 0] == 3.2,
  X[2, 1] == 5.9,
  x[0] * x[1] * x[2] == 1.0,
]
problem = cp.Problem(cp.Minimize(objective), constraints)
problem.solve(gp=True)

print("Optimal value:\n", 1.0/(m * n) * (problem.value - m * n), "\n")
print("Outer product approximation\n", outer_product.value, "\n")
print("x: ", x.value)
print("y: ", y.value)
```

    Optimal value:
     1.7763568394002505e-14

    Outer product approximation
     [[1.         1.84375    1.9       ]
     [0.43389831 0.8        0.82440678]
     [3.2        5.89999999 6.07999999]]

    x:  [0.89637009 0.38893346 2.86838428]
    y:  [1.11561063 2.0569071  2.1196602 ]


```python
import numpy as np
import scipy as sp
from scipy.integrate import solve_ivp
from scipy.optimize import minimize

from sympy import*
import sympy as sp

from sympy.interactive import printing
printing.init_printing(use_latex = True)

import matplotlib.pyplot  as plt
```

The net force for moving with air friction under gravity is

 $\vec{F_{net}} = \vec{F_{g}} + \vec{F_{f}} = -mg\hat{y} \ - b \mid \vec{\nu} \mid \vec{\nu} $

and noting that $\vec{v} = \dot{x} \hat{x} + \dot{y} \hat{y}$

or in vector form

$\vec{F_{net}} = \begin{bmatrix}
-b\dot{x}\sqrt{\dot{x}^{2} \ +\ \dot{y}^{2}}\\
-mg\ -b\dot{y}\sqrt{\dot{x}^{2} +\dot{y}^{2}}
\end{bmatrix}$

Using the fact

$\vec{F}_{net} = m \vec{a} = m \langle \ddot{x} \ddot{y}\rangle$

$m \begin{bmatrix}\ddot{x} \\ \ddot{y} \end{bmatrix} = \begin{bmatrix}
-b\dot{x}\sqrt{\dot{x}^{2} \ +\ \dot{y}^{2}}\\
-mg\ -b\dot{y}\sqrt{\dot{x}^{2} +\dot{y}^{2}}
\end{bmatrix}$

and thus have two coupled differential equations

$\ddot{x} = - \frac{b}{m} \dot{x} \sqrt{\dot{x}^2 + \dot{y}^2}$

$\ddot{y} = -g - \frac{b}{m} \dot{y} \sqrt{\dot{x}^2 + \dot{y}^2}$

Defining $x' = \frac{x}{g} $ and $y' = \frac{y}{g} $ we get

$\ddot{x'} = -\frac{bg}{m} \dot{x'}\sqrt{\dot{x'}^2 + \dot{y'}^2}$

$\ddot{y'} = -1 - \frac{bg}{m} \dot{y'} \sqrt{\dot{x'}^2 + \dot{y'}^2}$

In python we can only solve ODEs, so defining $v_x = \dot{x}$ and $v_y = \dot{y}$ we get a system of 4 coupled first order ODEs

* $\dot{x} = v_x$
* $\dot{y} = v_y$
* $\dot{v_x} = -\textbf{B}\dot{x} \sqrt{\dot{x}^2 + \dot{y}^2}$
* $\dot{v_y} = -1 - \textbf{B}\dot{y} \sqrt{\dot{x}^2 + \dot{y}^2}$

Where $\textbf{B} = \frac{bg}{m}$

Define $\vec{S} = \langle x, v_x, y, v_y \rangle$. To solve ODEs in python, we need to write a fucntion that takes $\vec{S}$ and time t, and returns $d\vec{S}/dt$. In other words we want $f$ in

$ \frac{d\vec{S}}{dt} = f(\vec{S}, t) $

```python
#Define function f above

def dsdt(t, S, B):
  x, vx, y, vy = S
  return [vx, -B*np.sqrt(vx**2+vy**2)*vx, vy, -1-B*np.sqrt(vx**2+vy**2)*vy]
```

```python
 B = 10
 V = 1
 t1 = 40*np.pi / 180
 t2 = 45 * np.pi /180
 t3 = 50 * np.pi / 180
```

Solve the ODE using scipy- The fucntion takes in $d\vec{S}/dt$ function, time period to solve over [0, 2] seconds, initial conditions, and additional arguments B(friction force) for the fucntion

```python
sol1 = solve_ivp(dsdt, [0, 2], y0=[0,V*np.cos(t1),0,V*np.sin(t1)], t_eval=np.linspace(0,2,1000), args=(B,))
sol2 = solve_ivp(dsdt, [0, 2], y0=[0,V*np.cos(t2),0,V*np.sin(t2)], t_eval=np.linspace(0,2,1000), args=(B,))
sol3 = solve_ivp(dsdt, [0, 2], y0=[0,V*np.cos(t3),0,V*np.sin(t3)], t_eval=np.linspace(0,2,1000), args=(B,))
```

```python
#sol1.y is the list of all the solution to the diffrential equation
#not to be confused with the y we

plt.figure(figsize=(20,8))
plt.plot(sol1.y[0],sol1.y[2], label=r'$\theta_0=40^{\circ}$')
plt.plot(sol2.y[0],sol2.y[2], label=r'$\theta_0=45^{\circ}$')
plt.plot(sol3.y[0],sol3.y[2], label=r'$\theta_0=50^{\circ}$')
plt.ylim(bottom = 0)
plt.legend()
plt.xlabel('$x/g$', fontsize = 20)
plt.ylabel('$y/g$', fontsize = 20)
plt.show()
```

```python
def get_distance(angle, B= 0, V= 1, t= 2):
  v0x = V*np.cos(angle*np.pi/180)
  v0y = V*np.sin(angle*np.pi/180)
  sol = solve_ivp(dsdt, [0, t], y0=[0,v0x,0,v0y], t_eval=np.linspace(0,t,10000), args=(B,), atol=1e-7, rtol=1e-4)
  just_above_idx = np.where(np.diff(np.sign(sol.y[2])) < 0)[0][0]
  just_below_idx = just_above_idx + 1

  x_loc = (sol.y[0][just_above_idx] + sol.y[0][just_below_idx])/2
  return x_loc

```

```python
print(f'Launch angel 45 degrees distance travelled: {get_distance(45, B=0, V=1)}')
print(f'Launch angel 40 degrees distance travelled: {get_distance(40, B=0, V=1)}')
```

    Launch angel 45 degrees distance travelled: 1.0000197012460217
    Launch angel 40 degrees distance travelled: 0.9848486164910958

```python
angles = np.linspace(0, 90, 200)
x_locs = np.vectorize(get_distance)(angles, B= 1000, V= 1)
```

```python
plt.figure(figsize=(20,8))
plt.plot(angles, x_locs)
plt.xlabel('Launch Angle [degrees]', fontsize = 20)
plt.ylabel('Maximum Distance[distance/gravity]', fontsize = 20)
plt.axvline(angles[np.argmax(x_locs)], ls = '--', color = 'r')
plt.show()
print("Optimal Launch angle", angles[np.argmax(x_locs)])
```

```python
V1 = 1
V2 = 20
angles = np.linspace(35, 45, 200)
Bs =  np.linspace(0, 1, 50)

results_v1 = [np.vectorize(get_distance)(angles, B=B, V=V1) for B in Bs]
opt_angles_v1 = [angles[np.argmax(result)] for result in results_v1]
results_v2 = [np.vectorize(get_distance)(angles, B=B, V=V2, t=6) for B in Bs]
opt_angles_v2 = [angles[np.argmax(result)] for result in results_v2]
```

```python
plt.plot(Bs, opt_angles_v1, 'o--', label='$v_0/g=1$s')
#plt.plot(Bs, opt_angles_v2, 'o--', label='$v_0/g=2$s')
plt.legend(fontsize=17)
plt.xlabel('bg/m [1/$s^2$]', fontsize=20)
plt.ylabel('Optimal Angle', fontsize=20)
plt.grid()

```

```python

```


### Computing for Mathematics - 2021/2022 individual coursework

**Important** Do not delete the cells containing:

```
### BEGIN SOLUTION

### END SOLUTION
```

write your solution attempts in those cells.

To submit this notebook:

- Change the name of the notebook from `main` to: `<student_number>`. For example, if your student number is `c1234567` then change the name of the notebook to `c1234567`.
- **Write all your solution attempts in the correct locations**;
- **Do not delete** any code that is already in the cells;
- Save the notebook (`File>Save As`);
- Follow the instructions given to submit.

#### Question 1

(__Hint__: This question is similar to [the first exercise of the Matrices chapter](https://vknight.org/pfm/tools-for-mathematics/04-matrices/solutions/main.html#question-1) of Python for mathematics.)

For each of the following matrices **output** their determinant.

a. \\(\begin{pmatrix}4 & 4 & 4 \\ 0 & 2 & 0 \\ 12 & 12 & 12\end{pmatrix}\\)

Available marks: 1

```python
import sympy as sym

### BEGIN SOLUTION

### END SOLUTION
```

b. \\(\begin{pmatrix}3\end{pmatrix}\\)

_Available marks: 1_

```python
x = sym.Symbol("x")

### BEGIN SOLUTION

### END SOLUTION
```

c. \\(\begin{pmatrix}50 \pi & 40 e & 1 \\ 12 & 3 & 1 \\ -500 & 400 &
\pi ^e\end{pmatrix}\\)

_Available marks: 2_

```python
### BEGIN SOLUTION

### END SOLUTION
```

### Question 2

(__Hint__: This question is similar to the [second exercise of the Sequences chapter of Python for mathematics](https://vknight.org/pfm/tools-for-mathematics/07-sequences/solutions/main.html#question-2).)

Using recursion, create a function `get_sequence` which gives the terms of the following sequence:

\\[
\begin{cases}
a_0 &= -2\\
a_n & 4 a_{n-1}, n\geq 1
\end{cases}
\\]

_Available marks: 2_

```python
def get_sequence(n):
### BEGIN SOLUTION

### END SOLUTION
```

### Question 3

(__Hint__: This question is similar to the [fourth exercise of the Matrices chapter of Python for mathematics](https://vknight.org/pfm/tools-for-mathematics/04-matrices/solutions/main.html#question-4).)

The matrix \\(A\\) is given by: \\(
    \begin{pmatrix}
        4 & 3 & 2 & 3\\
        3 & 1 & 1 & 3\\
        0 & -1 & 2 & 1\\
        0 & 2 & 2 & 3\\
    \end{pmatrix}\\)

a. Create a variable `A_inv` which has value the inverse of `A`

_Available marks: 2_

```python
### BEGIN SOLUTION

### END SOLUTION
```

b. Create a variable `x_sol` which has value the vector
representing the solution to the following linear system:

\\[
  \begin{eqnarray}
    4 x_1 + 3 x_2 + 2 x_3 + 3 x_4 &= 2\\
    3 x_1 + x_2 + x_3 + 3 x_4 &= 7\\
    - x_2 + 2 x_3 + x_4 &= 7\\
    2 x_2 + 2 x_3 + 3 x_4 &= 0\\
  \end{eqnarray}
\\]

_Available marks: 2_

```python
### BEGIN SOLUTION

### END SOLUTION
```

### Question 4

(__Hint__: This question is similar to the [second exercise of the Algebra chapter of Python for mathematics](https://vknight.org/pfm/tools-for-mathematics/03-calculus/solutions/main.html#question-2))

Consider the function: \\(f(x)=\frac{\cos(x)}{x}\\).

a. Create a variable `expression` which has value: \\(\frac{f(x)- f(x-h)}{h}\\)

_available marks: 2_

```python
h = sym.Symbol("h")

### BEGIN SOLUTION

### END SOLUTION
```

b. Create a variable `limit` which has value: \\(\lim_{h\to 0}\frac{f(x) - f(x-h)}{h}\\)

_available marks: 1_

```python
### BEGIN SOLUTION

### END SOLUTION
```

```python
limit
```

$\displaystyle - \frac{x \sin{\left(x \right)} + \cos{\left(x \right)}}{x^{2}}$

c. Using this, output the value \\(\frac{df}{dx}\\) at \\(x=\pi\\).

*available marks: 1*

```python
import math

### BEGIN SOLUTION

### END SOLUTION
```


# Functions
So far in this course we've explored equations that perform algebraic operations to produce one or more results. A *function* is a way of encapsulating an operation that takes an input and produces exactly one ouput.

For example, consider the following function definition:

\begin{equation}f(x) = x^{2} + 2\end{equation}

This defines a function named ***f*** that accepts one input (***x***) and returns a single value that is the result calculated by the expression *x2 + 2*.

Having defined the function, we can use it for any input value. For example:

\begin{equation}f(3) = 11\end{equation}

You've already seen a few examples of Python functions, which are defined using the **def** keyword. However, the strict definition of an algebraic function is that it must return a single value. Here's an example of defining and using a Python function that meets this criteria:

```python
# define a function to return x^2 + 2
def f(x):
    return x**2 + 2

# call the function
f(3)
```

    11

You can use functions in equations, just like any other term. For example, consider the following equation:

\begin{equation}y = f(x) - 1\end{equation}

To calculate a value for ***y***, we take the ***f*** of ***x*** and subtract 1. So assuming that ***f*** is defined as previously, given an ***x*** value of 4, this equation returns a ***y*** value of **17** (*f*(4) returns 42 + 2, so 16 + 2 = 18; and then the equation subtracts 1 to give us 17). Here it is in Python:

```python
x = 4
y = f(x) - 1
print(y)
```

    17

Of course, the value returned by a function depends on the input; and you can graph this with the iput (let's call it ***x***) on one axis and the output (***f(x)***) on the other.

```python
%matplotlib inline

import numpy as np
from matplotlib import pyplot as plt

# Create an array of x values from -100 to 100
x = np.array(range(-100, 101))

# Set up the graph
plt.xlabel('x')
plt.ylabel('f(x)')
plt.grid()

# Plot x against f(x)
plt.plot(x,f(x), color='purple')

plt.show()
```

As you can see (if you hadn't already figured it out), our function is a *quadratic function* - it returns a squared value that results in a parabolic graph when the output for multiple input values are plotted.

## Bounds of a Function
Some functions will work for any input and may return any output. For example, consider the function ***u*** defined here:

\begin{equation}u(x) = x + 1\end{equation}

This function simply adds 1 to whatever input is passed to it, so it will produce a defined output for any value of ***x*** that is a *real* number; in other words, any "regular" number - but not an *imaginary* number like √-1, or ∞ (infinity). You can specify the set of real numbers using the symbol ${\rm I\!R}$ (note the double stroke). The values that can be used for ***x*** can be expressed as a *set*, which we indicate by enclosing all of the members of the set in "{...}" braces; so to indicate the set of all possible values for x such that x is a member of the set of all real numbers, we can use the following expression:

\begin{equation}\{x \in \rm I\!R\}\end{equation}

### Domain of a Function
We call the set of numbers for which a function can return value it's *domain*, and in this case, the domain of ***u*** is the set of all real numbers; which is actually the default assumption for most functions.

Now consider the following function ***g***:

\begin{equation}g(x) = (\frac{12}{2x})^{2}\end{equation}

If we use this function with an ***x*** value of **2**, we would get the output **9**; because (12 ÷ (2•2))2 is 9. Similarly, if we use the value **-3** for ***x***, the output will be **4**. However, what happens when we apply this function to an ***x*** value of **0**? Anything divided by 0 is undefined, so the function ***g*** doesn't work for an ***x*** value of 0.

So we need a way to denote the domain of the function ***g*** by indicating the input values for which a defined output can be returned. Specifically, we need to restrict ***x*** to a specific list of values - specifically any real number that is not 0. To indicate this, we can use the following notation:

\begin{equation}\{x \in \rm I\!R\;\;|\;\; x \ne 0 \}\end{equation}

This is interpreted as *Any value for x where x is in the set of real numbers such that x is not equal to 0*, and we can incorporate this into the function's definition like this:

\begin{equation}g(x) = (\frac{12}{2x})^{2}, \{x \in \rm I\!R\;\;|\;\; x \ne 0 \}\end{equation}

Or more simply:

\begin{equation}g(x) = (\frac{12}{2x})^{2},\;\; x \ne 0\end{equation}

When you plot the output of a function, you can indicate the gaps caused by input values that are not in the function's domain by plotting an empty circle to show that the function is not defined at this point:

```python
%matplotlib inline

# Define function g
def g(x):
    if x != 0:
        return (12/(2*x))**2

# Plot output from function g
import numpy as np
from matplotlib import pyplot as plt

# Create an array of x values from -100 to 100
x = range(-100, 101)

# Get the corresponding y values from the function
y = [g(a) for a in x]

# Set up the graph
plt.xlabel('x')
plt.ylabel('g(x)')
plt.grid()

# Plot x against g(x)
plt.plot(x,y, color='purple')

# plot an empty circle to show the undefined point
plt.plot(0,g(0.0000001), color='purple', marker='o', markerfacecolor='w', markersize=8)

plt.show()
```

Note that the function works for every value other than 0; so the function is defined for x = 0.000000001, and for x = -0.000000001; it only fails to return a defined value for exactly 0.

OK, let's take another example. Consider this function:

\begin{equation}h(x) = 2\sqrt{x}\end{equation}

Applying this function to a non-negative ***x*** value returns a meaningful output; but for any value where ***x*** is negative, the output is undefined.

We can indicate the domain of this function in its definition like this:

\begin{equation}h(x) = 2\sqrt{x}, \{x \in \rm I\!R\;\;|\;\; x \ge 0 \}\end{equation}

This is interpreted as *Any value for x where x is in the set of real numbers such that x is greater than or equal to 0*.

Or, you might see this in a simpler format:

\begin{equation}h(x) = 2\sqrt{x},\;\; x \ge 0\end{equation}

Note that the symbol ≥ is used to indicate that the value must be *greater than **or equal to*** 0; and this means that **0** is included in the set of valid values. To indicate that the value must be *greater than 0, **not including 0***, use the > symbol. You can also use the equivalent symbols for *less than or equal to* (≤) and *less than* (<).

When plotting a function line that marks the end of a continuous range, the end of the line is shown as a circle, which is filled if the function includes the value at that point, and unfilled if it does not.

Here's the Python to plot function ***h***:

```python
%matplotlib inline

def h(x):
    if x >= 0:
        import numpy as np
        return 2 * np.sqrt(x)

# Plot output from function h
import numpy as np
from matplotlib import pyplot as plt

# Create an array of x values from -100 to 100
x = range(-100, 101)

# Get the corresponding y values from the function
y = [h(a) for a in x]

# Set up the graph
plt.xlabel('x')
plt.ylabel('h(x)')
plt.grid()

# Plot x against h(x)
plt.plot(x,y, color='purple')

# plot a filled circle at the end to indicate a closed interval
plt.plot(0, h(0), color='purple', marker='o', markerfacecolor='purple', markersize=8)

plt.show()
```

Sometimes, a function may be defined for a specific *interval*; for example, for all values between 0 and 5:

\begin{equation}j(x) = x + 2,\;\; x \ge 0 \text{ and } x \le 5\end{equation}

In this case, the function is defined for ***x*** values between 0 and 5 *inclusive*; in other words, **0** and **5** are included in the set of defined values. This is known as a *closed* interval and can be indicated like this:

\begin{equation}\{x \in \rm I\!R\;\;|\;\; 0 \le x \le 5 \}\end{equation}

It could also be indicated like this:

\begin{equation}\{x \in \rm I\!R\;\;|\;\; [0,5] \}\end{equation}

If the condition in the function was **x > 0 and x < 5**, then the interval would be described as *open* and 0 and 5 would *not* be included in the set of defined values. This would be indicated using one of the following expressions:

\begin{equation}\{x \in \rm I\!R\;\;|\;\; 0 \lt x \lt 5 \}\end{equation}
\begin{equation}\{x \in \rm I\!R\;\;|\;\; (0,5) \}\end{equation}

Here's function ***j*** in Python:

```python
%matplotlib inline

def j(x):
    if x >= 0 and x <= 5:
        return x + 2

# Plot output from function j
import numpy as np
from matplotlib import pyplot as plt

# Create an array of x values from -100 to 100
x = range(-100, 101)
y = [j(a) for a in x]

# Set up the graph
plt.xlabel('x')
plt.ylabel('j(x)')
plt.grid()

# Plot x against k(x)
plt.plot(x, y, color='purple')

# plot a filled circle at the ends to indicate an open interval
plt.plot(0, j(0), color='purple', marker='o', markerfacecolor='purple', markersize=8)
plt.plot(5, j(5), color='purple', marker='o', markerfacecolor='purple', markersize=8)

plt.show()
```

Now, suppose we have a function like this:

\begin{equation}
k(x) = \begin{cases}
  0, & \text{if } x = 0, \\
  1, & \text{if } x = 100
\end{cases}
\end{equation}

In this case, the function has highly restricted domain; it only returns a defined output for 0 and 100. No output for any other ***x*** value is defined. In this case, the set of the domain is:

\begin{equation}\{0,100\}\end{equation}

Note that this does not include all real numbers, it only includes 0 and 100.

When we use Python to plot this function, note that it only makes sense to plot a scatter plot showing the individual values returned, there is no line in between because the function is not continuous between the values within the domain.

```python
%matplotlib inline

def k(x):
    if x == 0:
        return 0
    elif x == 100:
        return 1

# Plot output from function k
from matplotlib import pyplot as plt

# Create an array of x values from -100 to 100
x = range(-100, 101)
# Get the k(x) values for every value in x
y = [k(a) for a in x]

# Set up the graph
plt.xlabel('x')
plt.ylabel('k(x)')
plt.grid()

# Plot x against k(x)
plt.scatter(x, y, color='purple')

plt.show()
```

### Range of a Function
Just as the domain of a function defines the set of values for which the function is defined, the *range* of a function defines the set of possible outputs from the function.

For example, consider the following function:

\begin{equation}p(x) = x^{2} + 1\end{equation}

The domain of this function is all real numbers. However, this is a quadratic function, so the output values will form a parabola; and since the function has no negative coefficient or constant, it will be an upward opening parabola with a vertex that has a y value of 1.

So what does that tell us? Well, the minimum value that will be returned by this function is 1, so it's range is:

\begin{equation}\{p(x) \in \rm I\!R\;\;|\;\; p(x) \ge 1 \}\end{equation}

Let's create and plot the function for a range of ***x*** values in Python:

```python
%matplotlib inline

# define a function to return x^2 + 1
def p(x):
    return x**2 + 1

# Plot the function
import numpy as np
from matplotlib import pyplot as plt

# Create an array of x values from -100 to 100
x = np.array(range(-100, 101))

# Set up the graph
plt.xlabel('x')
plt.ylabel('p(x)')
plt.grid()

# Plot x against f(x)
plt.plot(x,p(x), color='purple')

# Plot Axis x and y
plt.axhline()
plt.axvline()

plt.show()
```

Note that the ***p(x)*** values in the plot drop exponentially for ***x*** values that are negative, and then rise exponentially for positive ***x*** values; but the minimum value returned by the function (for an *x* value of 0) is **1**.

MathsToML/Module01-Equations, Graphs, and Functions/01-08-Functions.ipynb