# Convex approximate dynamic programming

We consider a stochastic control problem of the form
\begin{equation}
\begin{array}{ll}
\mbox{minimize} & \underset{T \to \infty}\lim {\mathbb E} \left[\frac{1}{T} \sum_{t=0}^{T-1} \|{x_t}\|_2^2 + \|{\phi(x_t)}\|_2^2\right]\\[.2cm]
\mbox{subject to} & x_{t+1} = Ax_t + B\phi(x_t) + \omega_t,
\end{array}
\label{eq:adp}
\end{equation}
where $x_t\in\mathbf{R}^n$ is the state, $\phi:\mathbf{R}^n \to \mathcal U \subseteq \mathbf{R}^m$ is
the policy, $\mathcal U$ is a convex set representing the allowed set of controls,
and $\omega_t\in\Omega$ is a (random, i.i.d.) disturbance.
Here the variable is the policy $\phi$, and the expectation is taken over
disturbances and the initial state $x_0$. If $\mathcal U$ is not an affine
set, then this problem is in general very difficult to solve.

A common heuristic for solving stochastic control problems is
approximate dynamic programming (ADP), which parametrizes $\phi$
and replaces the minimization over functions $\phi$ with a minimization over parameters.
In this example, we take $\mathcal U$ to be the unit ball and we represent $\phi$
as a particular quadratic *control-Lyapunov* policy.
Evaluating $\phi$ corresponds to solving the SOCP
\begin{equation}
\begin{array}{ll}
\mbox{minimize} & u^T P u + x_t^T Q u + q^T u \\
\mbox{subject to} & \|{u}\|_2 \leq 1,
\end{array}
\label{eq:policy}
\end{equation}
with variable $u$ and parameters $P$, $Q$, $q$, and $x_t$. We can run
gradient descent (SGD) on $P$, $Q$, and $q$ to
approximately solve the original problem, which requires requires differentiating
through the quadratic policy. Note that if $u$ were unconstrained, the original problem
could be solved exactly, via linear quadratic regulator (LQR) theory.

```python
import cvxpy as cp
import numpy as np
import torch
from scipy.linalg import solve_discrete_are
from scipy.linalg import sqrtm

from cvxpylayers.torch import CvxpyLayer

# Generate data
torch.manual_seed(1)
np.random.seed(1)

n = 2
m = 3

A = np.eye(n) + 1e-2 * np.random.randn(n, n)
B = 1e-2 / 3 * np.random.randn(n, m)
Q = np.eye(n)
R = np.eye(m)

# Compute LQR control policy
P_lqr = solve_discrete_are(A, B, Q, R)
P = R + B.T@P_lqr@B
P_sqrt_lqr = sqrtm(P)

# Construct CVXPY problem and layer
x_cvxpy = cp.Parameter((n, 1))
P_sqrt_cvxpy = cp.Parameter((m, m))
P_21_cvxpy = cp.Parameter((n, m))
q_cvxpy = cp.Parameter((m, 1))

u_cvxpy = cp.Variable((m, 1))
y_cvxpy = cp.Variable((n, 1))

objective = .5 * cp.sum_squares(P_sqrt_cvxpy @ u_cvxpy) + x_cvxpy.T @ y_cvxpy + q_cvxpy.T @ u_cvxpy
problem = cp.Problem(cp.Minimize(objective), [cp.norm(u_cvxpy) <= 1, y_cvxpy == P_21_cvxpy @ u_cvxpy])
assert problem.is_dpp()
policy = CvxpyLayer(
    problem, [x_cvxpy, P_sqrt_cvxpy, P_21_cvxpy, q_cvxpy], [u_cvxpy])
```

Below, we train the policy and plot the estimated average cost for each iteration of
SGD for a numerical example, with $x \in
\mathbf{R}^2$ and $u \in \mathbf{R}^3$, a time horizon of $T=25$, and a batch
size of $8$. We initialize our policy's parameters with the LQR solution,
ignoring the constraint on $u$.

```python
import matplotlib.pyplot as plt

def train(iters):
    # Initialize with LQR control lyapunov function
    P_sqrt = torch.from_numpy(P_sqrt_lqr).requires_grad_(True)
    P_21 = torch.from_numpy(A.T @ P_lqr @ B).requires_grad_(True)
    q = torch.zeros((m, 1), requires_grad=True)
    variables = [P_sqrt, P_21, q]
    A_tch, B_tch, Q_tch, R_tch = map(torch.from_numpy, [A, B, Q, R])

    def g(x, u):
        return (x.t() @ Q_tch @ x + u.t() @ R_tch @ u).squeeze()

    def evaluate(x0, P_sqrt, P_21, q, T):
        x = x0
        cost = 0.
        for _ in range(T):
            u, = policy(x, P_sqrt, P_21, q)
            cost += g(x, u) / T
            x = A_tch @ x + B_tch @ u + .2 * torch.randn(n, 1).double()
        return cost

    def eval_loss(N=8, T=25):
        return sum([evaluate(torch.zeros(n, 1).double(), P_sqrt, P_21, q, T=T)
                    for _ in range(N)]) / N

    results = []
    optimizer = torch.optim.SGD(variables, lr=.02, momentum=.9)
    for i in range(iters):
        # use same seeds each iteration to get pretty training plot
        torch.manual_seed(1)
        np.random.seed(1)
        optimizer.zero_grad()
        loss = eval_loss()
        loss.backward()
        optimizer.step()
        results.append(loss.item())
        print("(iter %d) loss: %g " % (i, results[-1]))
    return results

results = train(iters=100)
```

    (iter 0) loss: 1.51906
    (iter 1) loss: 1.51596
    (iter 2) loss: 1.51004
    (iter 3) loss: 1.50076
    (iter 4) loss: 1.48964
    (iter 5) loss: 1.47732
    (iter 6) loss: 1.46409
    (iter 7) loss: 1.44985
    (iter 8) loss: 1.43349
    (iter 9) loss: 1.41393
    (iter 10) loss: 1.39383
    (iter 11) loss: 1.37273
    (iter 12) loss: 1.35031
    (iter 13) loss: 1.32644
    (iter 14) loss: 1.2979
    (iter 15) loss: 1.26749
    (iter 16) loss: 1.23994
    (iter 17) loss: 1.21162
    (iter 18) loss: 1.18454
    (iter 19) loss: 1.15832
    (iter 20) loss: 1.13336
    (iter 21) loss: 1.10918
    (iter 22) loss: 1.08801
    (iter 23) loss: 1.07002
    (iter 24) loss: 1.05461
    (iter 25) loss: 1.04056
    (iter 26) loss: 1.02755
    (iter 27) loss: 1.0159
    (iter 28) loss: 1.00493
    (iter 29) loss: 0.995569
    (iter 30) loss: 0.987571
    (iter 31) loss: 0.980465
    (iter 32) loss: 0.974158
    (iter 33) loss: 0.968241
    (iter 34) loss: 0.962846
    (iter 35) loss: 0.958006
    (iter 36) loss: 0.953747
    (iter 37) loss: 0.949987
    (iter 38) loss: 0.946659
    (iter 39) loss: 0.943703
    (iter 40) loss: 0.94107
    (iter 41) loss: 0.938717
    (iter 42) loss: 0.936607
    (iter 43) loss: 0.934709
    (iter 44) loss: 0.932997
    (iter 45) loss: 0.931448
    (iter 46) loss: 0.930042
    (iter 47) loss: 0.928762
    (iter 48) loss: 0.927593
    (iter 49) loss: 0.926524
    (iter 50) loss: 0.925542
    (iter 51) loss: 0.924639
    (iter 52) loss: 0.923806
    (iter 53) loss: 0.923036
    (iter 54) loss: 0.922322
    (iter 55) loss: 0.921659
    (iter 56) loss: 0.921042
    (iter 57) loss: 0.920465
    (iter 58) loss: 0.919926
    (iter 59) loss: 0.919421
    (iter 60) loss: 0.918946
    (iter 61) loss: 0.918499
    (iter 62) loss: 0.918078
    (iter 63) loss: 0.917679
    (iter 64) loss: 0.917302
    (iter 65) loss: 0.916944
    (iter 66) loss: 0.916604
    (iter 67) loss: 0.916281
    (iter 68) loss: 0.915972
    (iter 69) loss: 0.915678
    (iter 70) loss: 0.915396
    (iter 71) loss: 0.915126
    (iter 72) loss: 0.914867
    (iter 73) loss: 0.914618
    (iter 74) loss: 0.91438
    (iter 75) loss: 0.914149
    (iter 76) loss: 0.913928
    (iter 77) loss: 0.913714
    (iter 78) loss: 0.913507
    (iter 79) loss: 0.913307
    (iter 80) loss: 0.913114
    (iter 81) loss: 0.912927
    (iter 82) loss: 0.912745
    (iter 83) loss: 0.912569
    (iter 84) loss: 0.912398
    (iter 85) loss: 0.912232
    (iter 86) loss: 0.91207
    (iter 87) loss: 0.911913
    (iter 88) loss: 0.91176
    (iter 89) loss: 0.91161
    (iter 90) loss: 0.911465
    (iter 91) loss: 0.911323
    (iter 92) loss: 0.911185
    (iter 93) loss: 0.911049
    (iter 94) loss: 0.910917
    (iter 95) loss: 0.910788
    (iter 96) loss: 0.910662
    (iter 97) loss: 0.910539
    (iter 98) loss: 0.910418
    (iter 99) loss: 0.910299

```python
plt.figure()
plt.plot(results)
plt.xlabel('iteration')
plt.ylabel('average cost')
plt.savefig("adp.pdf")
plt.show()
```

```python

```


# 2. Unbounded Grids: the Semidiscrete Fourier Transform

## Aliasing

```julia
h = 0.25
k1 = π/(4h)
k2 = 9π/(4h)
```

```julia
using PyPlot
```

```julia
x0 = -1:h:1
xx = -1:0.01:1

y01 = sin.(k1.*x0)
y02 = sin.(k2.*x0)
yy1 = sin.(k1.*xx)
yy2 = sin.(k2.*xx);

```

```julia
plot(xx, yy1 )
plot(xx, yy2)
plot(x0, y01, "bo")
```

## Função `sinc`

$$
\text{sinc}(x) = \frac{\sin\pi x}{\pi x}
$$

```julia
xx = -15:0.01:15
yy = sinc.(xx)
plot(xx, yy, color="k")
axhline(0, ls="--", color="k")
for i in 1:15
    axvline(i, ls=":", color="r")
    axvline(-i, ls=":", color="b")
end
```

## Program 3

```julia
fun1(x) = float(x==0)
fun2(x) = float(abs(x) <= 3)
fun3(x) = max(0.0, 1-abs(x)/3)

function plotfun(plt, h, xmax, f)
    x = -xmax:h:xmax
    xx = -xmax-h/20:h/40:xmax+h/20

    v = f.(x)

    subplot(3,1,plt)
    plot(x, v, "k.")
    nxx = length(xx)
    nx = length(x)
    p = zeros(nxx)
    for i = 1:nx
        p .= p .+ v[i] .* sinc.((xx .- x[i])./h)
    end

    plot(xx, p, "k")
end

```

```julia
plotfun(1, 1.0, 10.0, fun1); grid(ls=":")
plotfun(2, 0.5, 10.0, fun2); grid(ls=":")
plotfun(3, 1.0, 10.0, fun3); grid(ls=":")

```

# Exercises

## Exercise 2.1

Let $\mathcal{F}$ denote the Fourier transform operator defined by (2.1), so that $u,v \in L^2(\mathbb{R})$ have transforms $\hat{u}=\mathcal{F}\{u\}$, $\hat{v}=\mathcal{F}\{v\}$. Verify the following properties.

### (a) Linearity
$$
\mathcal{F}\{u+v\} = \int_{-\infty}^\infty e^{-ikx}\left[u(x) + v(x)\right]\:dx = \int_{-\infty}^\infty\left[e^{-ikx}u(x) + e^{-ikx}v(x)\right]\:dx = \int_{-\infty}^\infty e^{-ikx}u(x) \:dx + \int_{-\infty}^\infty e^{-ikx}v(x) \:dx = \mathcal{F}\{u\} + \mathcal{F}\{v\} = \hat{u}(k) + \hat{v}{k}
$$

$$
\mathcal{F}\{cu\} = \int_{-\infty}^\infty e^{-ikx}\left[cu(x)\right]\:dx = c \int_{-\infty}^\infty e^{-ikx}u(x)\:dx =  c\mathcal{F}\{u\} = c\hat{u}(k)
$$

### (b) Translation
Substitute $y = x + x_0$, so that $dy= dx$ and $x = y-x_0$
$$
\mathcal{F}\{u(x+x_0)\} = \int_{-\infty}^\infty e^{-ikx}u(x+x_0)\:dx = \int_{-\infty}^\infty e^{-ik(y - x_0)}u(y)\:dy = \int_{-\infty}^\infty e^{i k x_0}e^{-iky}u(y)\:dy = e^{i k x_0} \mathcal{F}\{u\} =  e^{i k x_0} \hat{u}(k)
$$

### (c) Modulation

\begin{align}
\mathcal{F}\{e^{ik_0x}u(x)\} = \int_{-\infty}^\infty e^{-ikx}e^{ik_0x}u(x)\:dx &=  \int_{-\infty}^\infty e^{-ikx + ik_0x}u(x)\:dx = \\ =  \int_{-\infty}^\infty e^{-i(k-k_0)x}u(x)\:dx  &=   \hat{u}(k-k_0)
\end{align}

### (d) Dilation

Substitute $y = cx$ so that $dy= c dx$ and $x = y/c$

\begin{align}
\mathcal{F}\{u(cx)\} = \int_{-\infty}^\infty e^{-ikx}u(cx)\:dx =  \int_{-\infty}^\infty e^{-iky/c}u(y)\:\frac{dy}{c} =    \frac{1}{c}\hat{u}\left(\frac{k}{c}\right)
\end{align}

### (e) Conjugation

Substitute $y = cx$ so that $dy= c dx$ and $x = y/c$

\begin{align}
\mathcal{F}\{\overline{u(cx)}\} = \int_{-\infty}^\infty e^{-ikx}\overline{u(x)}\:dx &=  \int_{-\infty}^\infty \overline{e^{+ikx}u(x)}\:dx = \\
\overline{\int_{-\infty}^\infty e^{+ikx}u(x)\:dx} &= \overline{\hat{u}(-k)}
\end{align}

### Differentiation

$$
d( u e^{-ikx}) = u(x)(-ik) e^{-ikx}dx + u_x e^{-ikx}dx
$$

Therefore,
$$
u_x e^{-ikx} dx = d(ue^{-ikx}) + ik u(x) e^{-ikx}dx
$$

Substituting into the Fourier transform:

$$
\mathcal{F}\{u_x\} = \int_{-\infty}^\infty e^{-ikx}u_x\:dx =
\int_{-\infty}^\infty ik u(x) e^{-ikx}\:dx + \int_{-\infty}^\infty \:d\left(e^{-ikx}u\right)
$$

But for the Fourier transform to converge,

$$
\lim_{x\longrightarrow\pm\infty} u(x) = 0
$$

and we have
$$
\mathcal{F}\{u_x\} = \int_{-\infty}^\infty ik u(x) e^{-ikx}\:dx = ik\hat{u}(k)
$$

### (g) Inversion

$$
\mathcal{F}^{-1}\{\hat{u}\}(x)= \frac{1}{2\pi}\int_{\infty}^{\infty} \hat{u}(k) e^{ikx}\:dk
$$

$$
\mathcal{F}^{-1}\{u\}(x)= \frac{1}{2\pi}\int_{\infty}^{\infty} u(k) e^{ikx}\:dk
$$
Changing the names of the variables, $x\longrightarrow k$ and $k\longrightarrow x$,
$$
\mathcal{F}^{-1}\{u\}(k)= \frac{1}{2\pi}\int_{\infty}^{\infty} u(x) e^{ikx}\:dx = \frac{1}{2\pi} \hat{u}(-k)
$$

## Exercise 2.2

Let $u \in L^2(\mathbb{R})$ hava Fourier transform $\hat{u}$. Verify the following identities. *( a hermitian (skew-hermitian) function $f(x)$ is one with $f(-x) = \overline{f(x)}$ ($f(-x) = - \overline{f(x)}$).)*

### (a) $u(x)$ is even (odd) $\Longleftrightarrow$ $\hat{u}(k)$ is even (odd)

$u(x)$ is even

$$
\hat{u}(-k) = \int_{-\infty}^\infty e^{ikx} u(x)\:dx =
\int_{-\infty}^\infty e^{ikx} u(-x)\:dx = \\
-\int_{y=+\infty}^{y=-\infty} e^{-iky} u(y)\:dy =
\int_{-\infty}^\infty e^{-ikx} u(x)\:dx = \hat{u}(k)
$$

$\hat{u}(k)$ is even

$$
u(-x) = \frac{1}{2\pi} \int_{-\infty}^\infty e^{-ikx} \hat{u}(k)\:dk =
\frac{1}{2\pi} \int_{-\infty}^\infty e^{-ikx} \hat{u}(-k)\:dk = \\
\frac{-1}{2\pi} \int_{l=+\infty}^{-\infty} e^{ilx} \hat{u}(l)\:dl =
\frac{1}{2\pi} \int_{-\infty}^\infty e^{ikx} \hat{u}(k)\:dk = u(x)
$$

$u(x)$ is odd

$$
\hat{u}(-k) = \int_{-\infty}^\infty e^{ikx} u(x)\:dx =
-\int_{-\infty}^\infty e^{ikx} u(-x)\:dx = \\
+\int_{y=+\infty}^{y=-\infty} e^{-iky} u(y)\:dy =
-\int_{-\infty}^\infty e^{-ikx} u(x)\:dx = -\hat{u}(k)
$$

$\hat{u}(k)$ is odd

$$
u(-x) = \frac{1}{2\pi} \int_{-\infty}^\infty e^{-ikx} \hat{u}(k)\:dk =
-\frac{1}{2\pi} \int_{-\infty}^\infty e^{-ikx} \hat{u}(-k)\:dk = \\
+\frac{-1}{2\pi} \int_{l=+\infty}^{-\infty} e^{ilx} \hat{u}(l)\:dl =
-\frac{1}{2\pi} \int_{-\infty}^\infty e^{ikx} \hat{u}(k)\:dk = -u(x)
$$

### (b) $u(x)$ is real (imaginary) $\Longleftrightarrow$ $\hat{u}(k)$ is hermitian (skew-hermitian)

$u(x)$ is real

$$
\hat{u}(-k) = \int_{-\infty}^\infty e^{ikx} u(x)\:dx =
\int_{-\infty}^\infty \overline{e^{-ikx}} \overline{u(x)}\:dx =
\overline{\int_{-\infty}^\infty e^{-ikx} u(x)\:dx} = \overline{\hat{u}(k)}
$$

$\hat{u}(k)$ is hermitian

$$
u(x) = \frac{1}{2\pi}\int_{-\infty}^\infty \hat{u}(k) e^{ikx}\:dk =
\frac{1}{2\pi}\int_{-\infty}^\infty \overline{\hat{u}(-k)} \overline{e^{-ikx}} \:dk = \\
\overline{ \frac{1}{2\pi}\int_{k=-\infty}^{k=\infty} \hat{u}(-k) e^{-ikx} \:dk } = - \overline{ \frac{1}{2\pi}\int_{l=+\infty}^{l=-\infty} \hat{u}(l) e^{ilx} \:dl} = \\
\overline{ \frac{1}{2\pi}\int_{-\infty}^{\infty} \hat{u}(k) e^{ikx} \:dk} = \overline{u}(x)
$$

If $u(x) = \overline{u(x)}$, $u(x)$ is real.

$u(x)$ is imaginary

$$
\hat{u}(-k) = \int_{-\infty}^\infty e^{ikx} u(x)\:dx =
\int_{-\infty}^\infty \overline{e^{-ikx}} (-\overline{u(x)})\:dx =
-\overline{\int_{-\infty}^\infty e^{-ikx} u(x)\:dx} = -\overline{\hat{u}(k)}
$$

$\hat{u}(k)$ is skew-hermitian

$$
u(x) = \frac{1}{2\pi}\int_{-\infty}^\infty \hat{u}(k) e^{ikx}\:dk =
\frac{1}{2\pi}\int_{-\infty}^\infty \left[-\overline{\hat{u}(-k)} \right]\overline{e^{-ikx}} \:dk = \\
- \overline{ \frac{1}{2\pi}\int_{k=-\infty}^{k=\infty} \hat{u}(-k) e^{-ikx} \:dk } = + \overline{ \frac{1}{2\pi}\int_{l=+\infty}^{l=-\infty} \hat{u}(l) e^{ilx} \:dl} = \\
-\overline{ \frac{1}{2\pi}\int_{-\infty}^{\infty} \hat{u}(k) e^{ikx} \:dk} = -\overline{u}(x)
$$

If $u(x) = -\overline{u(x)}$, $u(x)$ is imaginary.

### (c) $u(x)$ is real and even $\Longleftrightarrow$ $\hat{u}(k)$ is real and even

 * $u(x)$ is real and even $\Longrightarrow$ $\hat{u}(k)$.

   From (a), $\hat{u}(k)$ is even. From (b), $\hat{u}(k)$ is hermitian. An even hermitian funcion is real.

 * $\hat{u}(k)$ is real and even $\Longrightarrow$ $u(x)$.

   From (a), $u(x)$ is even. From (b) $u(x)$ is real.

### (d) $u(x)$ is real and odd $\Longleftrightarrow$ $\hat{u}{k}$ is imaginary and odd

 * $u(x)$ is real and odd $\Longrightarrow$ $\hat{u}(k)$.

   From (a), $\hat{u}(k)$ is odd.
   From (b), $\hat{u}(k)$ is skew-hermitian and and odd and skew-hermitian function is imaginary.

 * $\hat{u}(k)$ is imaginary and odd $\Longrightarrow$ $u(x)$.

   From (a), $u(x)$ is odd. Since $\hat{u}(k)$ is odd and imaginary, it is is skew-hermitian and therefore, $u(x)$ is imaginary

### (e) $u(x)$ is imaginary and even $\Longleftrightarrow$ $\hat{u}(k)$ is imaginary and even

 * $u(x)$ is imaginary and even $\Longrightarrow$ $\hat{u}(k)$.

   From (a), $\hat{u}(k)$ is even. From (b), $\hat{u}(k)$ is skew-hermitian. An even skew-hermitian even funcion can only be imaginary

 * $\hat{u}(k)$ is imaginary and even $\Longrightarrow$ $u(x)$.

   From (a), $u(x)$ is even. From (b) $u(x)$ is imaginary.

### (f) $u(x)$ is imaginary and odd $\Longleftrightarrow$ $\hat{u}(k)$ is real and odd

 * $u(x)$ is imaginary and odd $\Longrightarrow$ $\hat{u}(k)$.

   From (a), $\hat{u}(k)$ is odd. From (b), $\hat{u}(k)$ is skew-hermitian. An odd skew-hermitian funcion can only be real

 * $\hat{u}(k)$ is real and odd $\Longrightarrow$ $u(x)$.

   From (a), $u(x)$ is odd. Buta an odd and real function is skew-hermitian and therefore $u(x)$ is imaginary.

## Exercise 2.3
Execute the command plot(sin.(1:3000),".") in Julia . What do you see? What does this have to do with aliasing? Give a quantitative answer, explaining exactly what frequency is being aliased by your eye and brain to what other frequency. Then, for fun, replace 3000 by 1000 to get a gure somewhat harder to explain. (This problem comes from Str91).

```julia
plot(sin.(1:3000), ".")
```

```julia
plot(sin.(1:1000), ".")
```

## Exercise 2.4

Derive 2.12 and 2.13.

 ### Derive 2.12

 $$
 S_h'(x_j) = \left\{
 \begin{matrix}
 0 & j=0\\
 \frac{(-1)^j}{jh} &  j\ne 0\\
 \end{matrix}
 \right.
 $$

 Quotient rule:

 $$
 \frac{d}{dx}\left(\frac{f}{g}\right) = \frac{g\cdot df/dx - f \cdot dg/dx}{g^2}
 $$
 where
 $$
 f(x) = \sin\left(\frac{\pi x}{h}\right) \qquad g(x) = \frac{\pi x}{h}
 $$

 Substituting $y = \pi x/h$,
  $$
  S'_h(x) = \frac{d S_h}{dy}\cdot \frac{dy}{dx} = \frac{\pi}{h}\frac{d}{dy}\left[\frac{sin(y)}{y}\right] =
 \frac{\pi}{h} \frac{y\cos y - \sin y}{y^2}
  $$

  But $x_j = \pi j h$ and therefore $y_j = \pi j$. For $x_j \ne 0$:
  $$
  S'_h(x_j) = \frac{\pi}{h} \frac{\pi j\cos \pi j - \sin \pi j}{\pi^2 j^2} = \frac{(-1)^j}{j h}
  $$

  For $x_j = 0$ we must use L'Hôpital's rule we arrive at
  $$
  S'_h(0) = 0
  $$

```julia

```


```python
import matplotlib.pyplot as plt
from mpl_toolkits import mplot3d
import numpy as np
import sympy as sym
```

```python
def f(X, n, q, b, c):
    Z = np.zeros(len(X))
    for i in range(len(X)):
        for j in range(int(n)):
            for k in range(int(n)):
                Z[i] += q[j][k]*X[i][j]*X[i][k]

        for j in range(int(n)):
            Z[i] += b[j]*X[i][j]
    return Z
```

```python
def f_mesh(X, Y, q, b, c):
    Z = np.zeros(len(X))
    Z = q[0][0]*X*X + q[0][1]*X*Y + q[1][0]*Y*X + q[1][1]*Y*Y + b[0]*X + b[1]*Y + c
    return Z
```

```python
def f2(X, Y, n, q, b, c):
    Z = q[0][0]*X*X + q[0][1]*X*Y + q[1][0]*Y*X + q[1][1]*Y*Y + b[0]*X + b[1]*Y + int(c[0])
    return Z
```

```python
def grad_descent(X, X1, Y1, Y, n, q, b, c, eps=0.05, precision=0.0001, max_iter=200):
    X_old = np.zeros((1, 2))
    X_new = np.zeros((1, 2))
    dfr = np.zeros((1, 2))

    X_new[0][0] = 4.9
    X_new[0][1] = 4.9
    i = 0
    Xs = np.zeros((max_iter,2))
    Ys = np.zeros(max_iter)

    x, y = sym.symbols('x y')
    df1 = sym.diff(f2(x, y, n, q, b, c), x)
    df2 = sym.diff(f2(x, y, n, q, b, c), y)

    # print("df1: {}, df2: {}".format(df1, df2))

    while np.all(abs(X_new - X_old)) > precision and max_iter > i:
        Xs[i] = X_new
        Ys[i] = f2 (X_new[0][0], X_new[0][1], n, q, b, c)

        X_old = X_new

        dfr[0][0] = df1.evalf(subs={x: X_old[0][0], y: X_old[0][1]})
        dfr[0][1] = df2.evalf(subs={x: X_old[0][0], y: X_old[0][1]})

        X_new = X_old - eps * dfr

        # print("X_new: {}, X_old: {}, dfr: {}".format(X_new, X_old, dfr))

        i += 1
        eps *= 0.99

    print("Finished with {} step".format(i))
    if (i < max_iter):
        Xs[i] = X_new
        Ys[i] = f2(X_new[0][0], X_new[0][1], n, q, b, c)

        for j in range(max_iter - 1, i, -1):
            Xs = np.delete(Xs, j, axis=0)
            Ys = np.delete(Ys, j, axis=0)

    return Xs, Ys
```

```python
X1 = np.arange(-5, 5, 0.1)
Y1 = np.arange(-5, 5, 0.1)
Z1 = np.zeros(len(X1))

X_new = np.zeros((100,2))

for i in range(len(X1)):
    X_new[i][0] = X1[i]
    X_new[i][1] = Y1[i]

Z1 = f(X_new, n, q, b, c)

x_list, y_list = grad_descent(X_new, X1, Y1, Z1, n, q, b, c)
```

    Finished with 200 step

```python
n = input("Enter power of your function: ")
q = []
for i in range(int(n)):
    q.append(input("Enter the function's coefficient matrix q's row q[{}]: ".format(i)).split())
b = input("Enter the function's coefficient b matrix: ").split()
c = input("Enter the function's constant c: ")

for i in range(int(n)):
    q[i] = list(map(float, q[i]))
b = list(map(float, b))
c = list(map(float, c))
```

    Enter power of your function: 2
    Enter the function's coefficient matrix q's row q[0]: 1 0
    Enter the function's coefficient matrix q's row q[1]: 0 2
    Enter the function's coefficient b matrix: 0 0
    Enter the function's constant c: 0

```python
X1, Y1 = np.meshgrid(X1, Y1)
Z1 = f_mesh(X1, Y1, q, b, c)

X, Y = zip(*x_list)
Z = y_list

ax = plt.subplots(nrows=1, ncols=1, figsize=(10,10))
cs = plt.contour(X1, Y1, Z1)
plt.clabel(cs, inline=1, fontsize=10)
colors = ['b', 'g', 'm', 'c', 'orange']
for j in range(1, len(X)):
    ax[1].annotate('', xy=(X[j], Y[j]), xytext=(X[j-1], Y[j-1]),
                arrowprops={'arrowstyle': '->', 'color': 'r', 'lw': 1},
                va='center', ha='center')
ax[1].scatter(X, Y, s=40, lw=0)
ax[1].set_xlabel('X')
ax[1].set_ylabel('Y')
ax[1].set_title('Minimizing function')

```

```python
fig = plt.figure()
ax = plt.axes(projection='3d')
ax.plot_surface(X1, Y1, Z1, rstride=1, cstride=1,
                cmap='viridis', edgecolor='none')
ax.set_title('surface')
plt.show()
```

```python
plt.figure()
cs = plt.contour(X1, Y1, Z1)
plt.clabel(cs, inline=1, fontsize=10)
plt.title('Two-dimensional contour plot')
plt.show()
```

```python

```


# Focused Gaussian beam waist

Given an optical imaging system with effective focal length $f$, how does the beam waist at the geometrical focus behave in dependence of the beam wave of the inciden Gaussian beam.

```python
import numpy as np
import mpmath as mp

from optics.beam import GaussianBeam
from optics.geometrical import ThinLens, Propagation
from optics.diffraction import UreyDiffraction, LiDiffraction

from matplotlib import pyplot as plt

colors = plt.get_cmap('tab20c').colors
```

These are the fixed system parameters.

```python
# aperture radius
a = 20e-3

# focal length
f = 25e-3

# wavelength
λ = 532e-9

# numerical aperture
NA = 0.8
```

## Beam waist at focal spot

In a first part we will estimate the spot size radius of the focused Gaussian beams from different approximations.
The first approximation arises from the ray transfer matrix formalism [[1]]. The ray transfer matrix is constructed from a thin lens following a free space propagation.

[1]: https://en.wikipedia.org/wiki/Ray_transfer_matrix_analysis

```python
ML = ThinLens(f).abcd()

print(ML)
```

    [  1.0  0.0]
    [-40.0  1.0]

```python
MP = Propagation(f).abcd()

print(MP)
```

    [1.0  0.025]
    [0.0    1.0]

```python
M = MP * ML
```

Using the complex beam parameter, one can use the ray transfer matrix formalism for the propagation of Gaussian beams [[2]]. The complex beam parameter is defined by
$$q(z)=(z-z_0)+iz_R,$$
wherein $z_R$ denotes the Rayleigh length. The reciprocal of the complex beam parameter is often written in terms of the beam waist $w(z)$ and radius of curvature $R(z)$,
$$\frac{1}{q(z)}=\frac{1}{R(z)}-\frac{i\lambda}{\pi w^2(z)}.$$
According to [[1]] ray transfer matrix formalism can be used with Gaussian beams when the complex beam parameter is updated according to $$q^\prime=\frac{Aq+B}{Cq+D},$$ wherein $A,B,C,D$ denote the matrix elements of the transfer matrix of the optical system. In our implementation we return a new Gaussian beam which origin $z_0$ is shifted according to $\operatorname{Re}(q^\prime)$ in order to reflect the change in curvature. The beam waist, however, is replaced by its new value in $\operatorname{Im}(q^\prime)$.

[2]: https://en.wikipedia.org/wiki/Ray_transfer_matrix_analysis#Ray_transfer_matrices_for_Gaussian_beams

```python
def focus_waist_ray(waist):
    return GaussianBeam(waist=waist, wavelength=λ).propagate(M).waist()
```

Drege [[3]] reports the analytical expression,
$$
\frac{0.97\lambda}{4T \mathit{NA}}\sqrt{\frac{1}{1-\exp(-1/T^2)}-1},
$$
to describe the focus beam waist of aperture limited imaging, wherein $T=w/a$ denotes the truncation parameter as ratio between the beam waist and the aperture radius.

[3]: https://www.osapublishing.org/ao/abstract.cfm?uri=ao-39-27-4918

```python
def focus_waist_drege(waist):
    T = waist / a

    return 0.97 * λ * mp.sqrt(mp.exp(1) / (1 - mp.exp(-1 / T**2)) - 1) / (4 * NA * T)
```

Urey [[4]] reports empirical obtained approximations for an aperture limited beam waist
$$
\begin{align}
&\frac{\lambda}{4\mathit{NA}}\frac{1.27}{T} &(T<0.5) \\
&\frac{\lambda}{4\mathit{NA}}\left(1.654-\frac{0.105}{T}+\frac{0.28}{T^2}\right) &(T>0.4)
\end{align}.
$$

[4]: https://www.osapublishing.org/ao/abstract.cfm?uri=ao-43-3-620

```python
def focus_waist_urey1(waist):
    T = waist / a

    return 1.27 * λ / (4 * NA * T)

def focus_waist_urey2(waist):
    T = waist / a

    return λ / (4 * NA * T) * (1.654 - 0.105 / T + 0.28 / T**2)
```

We need to vectorize these functions to allow vector inputs for multiple waists.

```python
focus_waist_ray = np.vectorize(focus_waist_ray)
focus_waist_drege = np.vectorize(focus_waist_drege)
focus_waist_urey1 = np.vectorize(focus_waist_urey1)
focus_waist_urey2 = np.vectorize(focus_waist_urey2)
```

```python
w = np.linspace(1e-3, 20e-3, 100)

plt.figure(figsize=(12, 6))

plt.title(r'Incoming and focused Gaussian beam waist')
plt.xlabel(r'$w_0$ (\si{\milli\meter})')
plt.ylabel(r'${w_0}^\prime$ (\si{\micro\meter})')

plt.plot(w / 1e-3, focus_waist_ray(w) / 1e-6, label='Ray', color=colors[0])
plt.plot(w / 1e-3, focus_waist_drege(w) / 1e-6, label='Drege', color=colors[3])
plt.plot(w[w < 10e-3] / 1e-3, focus_waist_urey1(w[w < 10e-3]) / 1e-6, label=r'Urey $(T<0.5)$', color=colors[4])
plt.plot(w[w > 8e-3] / 1e-3, focus_waist_urey2(w[w > 8e-3]) / 1e-6, label=r'Urey $(T>0.4)$', color=colors[7])

plt.legend(loc='upper center', bbox_to_anchor=(.5, -0.12), ncol=4)
plt.show()
```

We conclude that the spot size near the focus spot decreases with increasing incident beam waist. As we strive for a small focus spot, we need to find out at what incident beam waist diffraction effects get dominant and limit our resolution.


# Vector Spaces
* What is a vector?
    * Vectors are not lists of numbers (necessarily)
* Children have an intuitive idea of what vectors are
* A first definition of a vector
    * Vectors are a quantity with direction and length
    * Examples: 5 meters (length) to your left (direction). 20 miles (length) north (direction)

```python
%matplotlib inline

from matplotlib import pyplot as plt
import matplotlib as mpl
import numpy as np
from ipywidgets import interact
from mpl_toolkits.mplot3d import Axes3D
from matplotlib.patches import ConnectionPatch

def create_func_vis(num_funcs=1):
    fig, axs = plt.subplots(1, num_funcs+1)
    for ax in axs:
        ax.set_xlim(-2, 2)
        ax.set_ylim(-2, 2)
        ax.set_xticks([])
        ax.set_yticks([])
        ax.set_aspect('equal', adjustable='box')
    for i in range(num_funcs):
        con = ConnectionPatch(xyA=(0,0), xyB=(0,0), coordsA="data", coordsB="data", axesA=axs[i], axesB=axs[i+1], arrowstyle="->", alpha=0.5)
        axs[i+1].add_artist(con)
    return fig, axs

def plot_vec(x, y, u, v, ax=plt, **kwargs):
    return ax.quiver(x, y, u, v, angles='xy', units="xy", scale=1, scale_units="xy", **kwargs)

def plot_vec3(x, y, z, u, v, w, **kwargs):
    return plt.quiver(x, y, z, u, v, w, **kwargs)

def plot_vec_at_origin(vecs: np.ndarray, color='black', ax=plt):
    if len(vecs.shape) == 1:
        vecs = vecs.reshape(1, 2)
    o = np.zeros(vecs.shape[0])
    return ax.quiver(o, o, vecs[:, 0], vecs[:, 1], color=color, angles='xy', units="xy", scale=1, scale_units="xy")

def plot_vec3_at_origin(ax, vecs: np.ndarray, color='black'):
    if len(vecs.shape) == 1:
        vecs = vecs.reshape(1, 3)
    o = np.zeros(vecs.shape[0])
    return ax.quiver(o, o, o, vecs[:, 0], vecs[:, 1], vecs[:, 2], color=color)

def rotation(theta):
    c, s = np.cos(np.radians(theta)), np.sin(np.radians(theta))
    return np.array(((c, -s), (s, c)))

def get_vec(angle, length):
    return rotation(angle)@np.array((length,0))

@interact(angle=(0, 360, 10))
def demonstrate_no_coord_grid(angle):
    plt.xlim(-2, 2)
    plt.ylim(-2, 2)
    plt.axis('off')
    plt.gca().set_aspect('equal', adjustable='box')
    rot = rotation(angle)
    vecs = np.array((rot @ np.array([1, 1]), rot @ np.array([0, -1])))
    plot_vec_at_origin(vecs)
    plt.plot(0, 0, 'o', color="black")
```

    interactive(children=(IntSlider(value=180, description='angle', max=360, step=10), Output()), _dom_classes=('w…

* Coordinate grids are extra structure that we will add in shortly. They are not necessarily part of vectors

* Because vectors have a length, they can be scaled up or down by an arbitrary real number
    * Negative scaling corresponds to opposite direction

```python
@interact(len=(-2, 2, 0.1))
def demonstrate_scaling(len):
    plt.xlim(-2, 2)
    plt.ylim(-2, 2)
    plt.axis('off')
    plt.gca().set_aspect('equal', adjustable='box')
    vecs = len * np.array([1, 1])
    plot_vec_at_origin(vecs)
    plt.plot(0, 0, 'o', color="black")
```

    interactive(children=(FloatSlider(value=0.0, description='len', max=2.0, min=-2.0), Output()), _dom_classes=('…

* Vectors can also be added together.
    * Adding a vector $a$ to a vector $b$ produces a new vector $a+b$ by putting the tail of the second vector at the tip of the first vector
    * This can be done in either order, producing a sort of parallelogram
        * This is called the commutative law
    * Example:

```python
print("Control the angle and size of the red arrow")
@interact(angle=(0, 360, 10), length=(0.1, 2, 0.1))
def demonstrate_vector_addition(angle, length):
    plt.figure()
    plt.xlim(-2, 2)
    plt.ylim(-2, 2)
    plt.axis('off')
    plt.gca().set_aspect('equal', adjustable='box')
    rot = rotation(angle)
    vecs = np.array((get_vec(angle, length), [1, 1]))
    plot_vec_at_origin(vecs, color=['red', "blue"])
    plot_vec_at_origin(vecs[0]+vecs[1])
    plot_vec(vecs[:, 0], vecs[:, 1], vecs[::-1, 0], vecs[::-1, 1], color=["blue", 'red'])
    plt.plot(0, 0, 'o', color="black")
```

    Control the angle and size of the red arrow

    interactive(children=(IntSlider(value=180, description='angle', max=360, step=10), FloatSlider(value=1.0, desc…

* There are other operations on vectors I won't be covering in detail today
    * Dot product, cross product, tensor product, outer product, clifford product, determinant, etc...

* I claim that anything which has an addition and a scaling operation can be thought of as a vector
    * In other words, given a real number, like $\pi=3.1415\dots$, you must be able to form the vector $\pi v$
    * And you must be able to add $u$ to $v$ to get $u+v$
    * Anything with these two operations we will call a **vector space**

* Some examples:
    * The set of arrows in the 2d plane, without any coordinate grid forms a vector space as we saw above
    * The set of lists of 3 numbers, called $\mathbb{R}^3$ is a vector space
    $$\begin{bmatrix}1 \\2 \\3\end{bmatrix} +\begin{bmatrix}4 \\5 \\6\end{bmatrix} =\begin{bmatrix}5 \\7 \\9\end{bmatrix}$$

    * More generally, the set of lists of $n$ numbers is called $\mathbb{R}^n$
    * The set of polynomials in a single variable is a vector space
        $(x^2+2x+3)+(4x^2+5x+6)=5x^2+7x+9$

# Linear Independence, Span, and Bases
* Now that we have a working definition of a vector (an element of some vector space), we can begin to introduce coordinate grids to things in the form of a basis
* In order to define a basis we need to define some other terms first

* Linear combinations
    * A linear combination of a set of vectors is a sum of the vectors, each with a scale factor applied.
    * A linear combination of the vectors $u$ and $v$ is written as $au+by$.
    * An example of a linear combination of vectors in $\mathbb{R}^3$:
    $$3\begin{bmatrix}1 \\0 \\0\end{bmatrix} +2\begin{bmatrix}0 \\1 \\0\end{bmatrix} +5\begin{bmatrix}0\\0\\1\end{bmatrix}=\begin{bmatrix}3 \\2 \\5\end{bmatrix}$$

* Linear independence
    * A set of vectors is **linearly independent** if it contains no redundant vectors
    * In other words, no vector in the set can be written as a linear combination of the others
    * Examples:

```python
def demonstrate_linear_independence_1():
    plt.figure()
    plt.xlim(-2, 2)
    plt.ylim(-2, 2)
    plt.axis('off')
    plt.gca().set_aspect('equal', adjustable='box')
    vecs = np.array(((1,0), (0, 1)))
    plot_vec_at_origin(vecs)
    return plt.gcf()
demonstrate_linear_independence_1()

def demonstrate_linear_independence_2():
    plt.figure()
    plt.xlim(-2, 2)
    plt.ylim(-2, 2)
    plt.axis('off')
    plt.gca().set_aspect('equal', adjustable='box')
    vecs = np.array(((1,1), (0, 1)))
    plot_vec_at_origin(vecs)
    return plt.gcf()
demonstrate_linear_independence_2()

def demonstrate_linear_independence_3():
    fig = plt.figure()
    ax = fig.add_subplot(111, projection='3d')
    ax.set_xlim([-2, 2])
    ax.set_ylim([-2, 2])
    ax.set_zlim([-2, 2])
    plot_vec3_at_origin(ax, np.array(((0, 1, 0), (1,0,0), (0, 0, 1))))
    return fig
demonstrate_linear_independence_3();

def demonstrate_linear_independence_4():
    fig = plt.figure()
    ax = fig.add_subplot(111, projection='3d')
    ax.set_xlim([-2, 2])
    ax.set_ylim([-2, 2])
    ax.set_zlim([-2, 2])
    plot_vec3_at_origin(ax, np.array(((0, 1, 1), (1,0,0), (0, 0, 1))))
    return fig
demonstrate_linear_independence_4();
```

* A set of vectors is linearly dependent if it contains extra, redundant vectors
    * In other words, some vector can be written as a linear combination of the other vectors
    * Examples:

```python
def demonstrate_linear_dependence_1():
    plt.figure()
    plt.xlim(-2, 2)
    plt.ylim(-2, 2)
    plt.axis('off')
    plt.gca().set_aspect('equal', adjustable='box')
    vecs = np.array(((2,0), (1,0)))
    plot_vec_at_origin(vecs, color=["red", "blue"])
    return plt.gcf()
demonstrate_linear_dependence_1();

def demonstrate_linear_dependence_2():
    plt.figure()
    plt.xlim(-2, 2)
    plt.ylim(-2, 2)
    plt.axis('off')
    plt.gca().set_aspect('equal', adjustable='box')
    vecs = np.array((get_vec(0, 1), [1, 1]))
    plot_vec_at_origin(vecs)
    plot_vec_at_origin(vecs[0]+vecs[1])
    plt.plot(0, 0, 'o', color="black")
demonstrate_linear_dependence_2();

def demonstrate_linear_dependence_3():
    fig = plt.figure()
    ax = fig.add_subplot(111, projection='3d')
    ax.set_xlim([-2, 2])
    ax.set_ylim([-2, 2])
    ax.set_zlim([-2, 2])
    plot_vec3_at_origin(ax, np.array(((1, 1, 0), (1,0,0), (0, 1, 0))))
    return fig
demonstrate_linear_dependence_3();
```

* Span
    * The next notion we need to define is that of a span of a set of vectors
    * Every vector in the plane can be written as a linear combination of the following two vectors:

```python
demonstrate_linear_independence_1();
```

* Next we demonstrate why this happens.
    * Note how you can stretch the blue and red vectors to reach any vector on the plane:

```python
@interact(l1=(-2, 2, 0.1), l2=(-2, 2, 0.1))
def demonstrate_span(l1, l2):
    plt.figure()
    plt.xlim(-2, 2)
    plt.ylim(-2, 2)
    plt.axis('off')
    plt.gca().set_aspect('equal', adjustable='box')
    vec1 = l1 * np.array((1,0))
    vec2 = l2 * np.array((1,1))
    plot_vec_at_origin(vec1, color='red')
    plot_vec(vec1[0], vec1[1], vec2[0], vec2[1], color="blue")
    plt.plot(0, 0, 'o', color="black")
```

    interactive(children=(FloatSlider(value=0.0, description='l1', max=2.0, min=-2.0), FloatSlider(value=0.0, desc…

* Given a set of vectors, the set of all linear combinations of those vectors is called the **span** of that set
* Example:
    * The span of the two vectors visualized on the next slide is the entire green plane

```python
@interact(angle=(0, 80, 10))
def demonstrate_span(angle):
    vec1 = np.array((1, 0, 0))
    vec2 = np.array((0, np.cos(np.radians(angle)), np.sin(np.radians(angle))))
    normal = np.cross(vec1, vec2)
    x, y = np.meshgrid(np.arange(-2, 2, 0.1), np.arange(-2, 2, 0.1))
    z = (-normal[0] * x - normal[1] * y) * 1. /normal[2]
    fig = plt.figure()
    ax = fig.add_subplot(111, projection='3d')
    ax.set_xlim([-2, 2])
    ax.set_ylim([-2, 2])
    ax.set_zlim([-2, 2])
    # display(np.stack(vec1, vec2))
    plot_vec3_at_origin(ax, np.stack((vec1, vec2)))
    art = ax.plot_surface(x, y, z, color = 'g')
    art.set_alpha(0.5)
    return fig
```

    interactive(children=(IntSlider(value=40, description='angle', max=80, step=10), Output()), _dom_classes=('wid…

* Basis
    * If a set of vectors is linearly independent, then it is said to be the basis for its span
    * We say that a basis is a linearly independent spanning set
    * Examples:
        * The two vectors above are a basis for the green plane
        * Any of the examples of linearly independent sets are also basis for the plane/space

* Rephrasing the definition:
    * A set of vectors is a basis for a vector space if *every vector* can be written *uniquely* as a linear combination of the basis vectors

* A stray definition
    * We won't use this definition today, but the number of vectors in every basis of a vector space is the same
    * This number is called the dimension of the vector space
    * The dimension of the vector space of arrows in the plane is 2, as you would expect
    * The dimension of $\mathbb{R}^n$ is $n$, as you would expect
    * The dimension of the set of all polynomials is infinite
        * Basis: $1, x, x^2, x^3, x^4, x^5, \dots$

* All of this together shows us that there are a lot of different bases for the same vector space
    * Bases are arbitrary, and our vector space of arrows doesn't come with a default choice
    * $\mathbb{R}^3$ does come with a canonical choice of basis though:
        $$\begin{bmatrix}1 \\0 \\0\end{bmatrix},\begin{bmatrix}0 \\1 \\0\end{bmatrix},\begin{bmatrix}0\\0\\1\end{bmatrix}$$

# Linear transformations
- Recall the definition of a function in math:
    - a function $f$ takes a vector $v$ and produces an output vector $f(v)$. The vector it produces is always the same given the same input vector
    - Functions have a **domain** and a **codomain**, representing the vector spaces that the input and output are in.
    - If the input space is $V$ and the output space is $W$ we write this as $f: V \to W$. We read this as "f is a function from V to W".
- A linear transform is a function satisfying an additional property: linearity
    - Preservation of addition: $f(u+v)=f(u)+f(v)$
    - Preservation of scaling: $f(cv)=cf(v)$
- Linearity is sort of like the distributive law and commutative law combined.
- Examples:

```python
@interact(l1=(-2, 2, 0.1), l2=(-2, 2, 0.1))
def demonstrate_linear_transform_1(l1, l2):
    fig, ax = create_func_vis()
    vec1 = l1 * np.array((1,0.5))
    vec2 = l2 * np.array((1,1))
    plot_vec_at_origin(vec1, color='red', ax=ax[0])
    plot_vec(vec1[0], vec1[1], vec2[0], vec2[1], color="blue", ax=ax[0])
    ax[0].plot(0, 0, 'o', color="black")

    vec1 = 2 * vec1
    vec2 = 2 * vec2

    plot_vec_at_origin(vec1, color='red', ax=ax[1])
    plot_vec(vec1[0], vec1[1], vec2[0], vec2[1], color="blue", ax=ax[1])
    ax[1].plot(0, 0, 'o', color="black")

@interact(l1=(-2, 2, 0.1), l2=(-2, 2, 0.1))
def demonstrate_linear_transform_2(l1, l2):
    fig, ax = create_func_vis()
    vec1 = l1 * np.array((1,0.5))
    vec2 = l2 * np.array((1,1))
    plot_vec_at_origin(vec1, color='red', ax=ax[0])
    plot_vec(vec1[0], vec1[1], vec2[0], vec2[1], color="blue", ax=ax[0])
    ax[0].plot(0, 0, 'o', color="black")

    rot = rotation(30)
    vec1 = rot @ vec1
    vec2 = rot @ vec2

    plot_vec_at_origin(vec1, color='red', ax=ax[1])
    plot_vec(vec1[0], vec1[1], vec2[0], vec2[1], color="blue", ax=ax[1])
    ax[1].plot(0, 0, 'o', color="black")

@interact(l1=(-2, 2, 0.1), l2=(-2, 2, 0.1))
def demonstrate_linear_transform_3(l1, l2):
    fig, ax = create_func_vis()
    vec1 = l1 * np.array((1,0))
    vec2 = l2 * np.array((0,1))
    plot_vec_at_origin(vec1, color='red', ax=ax[0])
    plot_vec(vec1[0], vec1[1], vec2[0], vec2[1], color="blue", ax=ax[0])
    ax[0].plot(0, 0, 'o', color="black")

    skew = np.array(((1,0), (1,1))).T
    vec1 = skew @ vec1
    vec2 = skew @ vec2

    plot_vec_at_origin(vec1, color='red', ax=ax[1])
    plot_vec(vec1[0], vec1[1], vec2[0], vec2[1], color="blue", ax=ax[1])
    ax[1].plot(0, 0, 'o', color="black")

@interact(l1=(-2, 2, 0.1), l2=(-2, 2, 0.1))
def demonstrate_linear_transform_4(l1, l2):
    fig, ax = create_func_vis()
    vec1 = l1 * np.array((1,0))
    vec2 = l2 * np.array((0,1))
    plot_vec_at_origin(vec1, color='red', ax=ax[0])
    plot_vec(vec1[0], vec1[1], vec2[0], vec2[1], color="blue", ax=ax[0])
    ax[0].plot(0, 0, 'o', color="black")

    ref = np.array(((-1,0), (0,1))).T
    vec1 = ref @ vec1
    vec2 = ref @ vec2

    plot_vec_at_origin(vec1, color='red', ax=ax[1])
    plot_vec(vec1[0], vec1[1], vec2[0], vec2[1], color="blue", ax=ax[1])
    ax[1].plot(0, 0, 'o', color="black")
```

    interactive(children=(FloatSlider(value=0.0, description='l1', max=2.0, min=-2.0), FloatSlider(value=0.0, desc…

    interactive(children=(FloatSlider(value=0.0, description='l1', max=2.0, min=-2.0), FloatSlider(value=0.0, desc…

    interactive(children=(FloatSlider(value=0.0, description='l1', max=2.0, min=-2.0), FloatSlider(value=0.0, desc…

    interactive(children=(FloatSlider(value=0.0, description='l1', max=2.0, min=-2.0), FloatSlider(value=0.0, desc…

- Freshman's dream
    - A common error by young or inexperience mathematical discipuli is to assume a nonlinear operation is linear. As an example:
    - assuming $(x+y)^n=x^n+y^n$
- Why is linearity important?
    - Lots of reasons: first of all a lot of important functions are linear
        - Rotations
        - Scaling
        - Skewing
        - Reflections
    - Second of all, they are simpler to deal with, as we shall soon see
    - Third, many functions can be approximated by linear functions. Ask if you want further details on this

- Composition of linear functions
    - Given any two functions, $f$ and $g$, you can form the composition of them: $f(g(v))$.
    - This means, first apply the transform $g$, then apply the transform $f$ to the output of it
    - Note that in order for functions to be composable their domains and codomains must match up like $f: V \to W$ and $g: U \to V$
    - The composition of two linear functions is linear:
        $$\begin{align}f(g(au+bv))&=f(g(au)+g(bv))\\&= f(ag(u)+bg(v)) \\&= af(g(u))+bf(g(v))\end{align}$$
    - Examples:

```python
@interact(l1=(-2, 2, 0.1), l2=(-2, 2, 0.1))
def demonstrate_composition(l1, l2):
    fig, ax = create_func_vis(num_funcs=2)
    vec1 = l1 * np.array((1,0))
    vec2 = l2 * np.array((0,1))

    plot_vec_at_origin(vec1, color='red', ax=ax[0])
    plot_vec(vec1[0], vec1[1], vec2[0], vec2[1], color="blue", ax=ax[0])
    ax[0].plot(0, 0, 'o', color="black")

    rot = rotation(30)
    vec1 = rot @ vec1
    vec2 = rot @ vec2

    plot_vec_at_origin(vec1, color='red', ax=ax[1])
    plot_vec(vec1[0], vec1[1], vec2[0], vec2[1], color="blue", ax=ax[1])
    ax[1].plot(0, 0, 'o', color="black")

    skew = np.array(((1,0), (1,1))).T
    vec1 = skew @ vec1
    vec2 = skew @ vec2

    plot_vec_at_origin(vec1, color='red', ax=ax[2])
    plot_vec(vec1[0], vec1[1], vec2[0], vec2[1], color="blue", ax=ax[2])
    ax[2].plot(0, 0, 'o', color="black")
```

    interactive(children=(FloatSlider(value=0.0, description='l1', max=2.0, min=-2.0), FloatSlider(value=0.0, desc…

- Isomorphism
    - If every vector in the codomain comes from a unique vector in the domain, we call this an isomorphism
    - Equivalently, an isomorphism is an invertible linear transform:
        - There exists an inverse function $f^{-1}$ such that $f(f^{-1}(v))=v$
    - Example:
        - The inverse of a 30 deg rotation is a -30 deg rotation

```python
@interact(l1=(-2, 2, 0.1), l2=(-2, 2, 0.1))
def demonstrate_inverse(l1, l2):
    fig, ax = create_func_vis(num_funcs=2)
    vec1 = l1 * np.array((1,0))
    vec2 = l2 * np.array((0,1))

    plot_vec_at_origin(vec1, color='red', ax=ax[0])
    plot_vec(vec1[0], vec1[1], vec2[0], vec2[1], color="blue", ax=ax[0])
    ax[0].plot(0, 0, 'o', color="black")

    rot = rotation(30)
    vec1 = rot @ vec1
    vec2 = rot @ vec2

    plot_vec_at_origin(vec1, color='red', ax=ax[1])
    plot_vec(vec1[0], vec1[1], vec2[0], vec2[1], color="blue", ax=ax[1])
    ax[1].plot(0, 0, 'o', color="black")

    rot = rotation(-30)
    vec1 = rot @ vec1
    vec2 = rot @ vec2

    plot_vec_at_origin(vec1, color='red', ax=ax[2])
    plot_vec(vec1[0], vec1[1], vec2[0], vec2[1], color="blue", ax=ax[2])
    ax[2].plot(0, 0, 'o', color="black")
```

    interactive(children=(FloatSlider(value=0.0, description='l1', max=2.0, min=-2.0), FloatSlider(value=0.0, desc…

- A choice of a basis for a vector space is the same as choosing an isomorphism to $\mathbb{R}^n$ (where $n$ is the dimension).
    - We can think of this function as assigning coordinates to every vector in our domain
    - Example:
        - Assign the vectors $\begin{bmatrix} 1 \\ 0 \end{bmatrix}, \begin{bmatrix} 0 \\ 1 \end{bmatrix}$ to the basis below

```python
demonstrate_linear_independence_1();
```

# Matrices
- Matrices are to basis as coordinates are to vector spaces
- Linear transforms are uniquely determined by their value on a basis
    - **Very Important**
    - Proof:
        - Let $x_1, x_2$ be a basis
        - Every vector $v$ can be written as $v=ax_1+bx_2$
            - We know this is possible since $x_1, x_2$ is a basis
        - Now, $f(v)=af(x_1)+bf(x_2)$
    - Therefore we can define any linear transform by just listing what the output of the basis vectors should be
        - So, if $f(x_1)=x_3$ and $f(x_2)=x_4$ then we write the linear transform as just $[x_3, x_4]$

- Remember that choosing a basis gives vectors coordinates
- If we choose a basis, we can write the output vectors in that basis
- So, if, in coordinates, $x_3=\begin{bmatrix}2 \\ 3 \end{bmatrix}$ and $x_4=\begin{bmatrix} 1 \\ 2 \end{bmatrix}$
    - then our matrix becomes $\begin{bmatrix} 2 & 1 \\ 3 & 2 \end{bmatrix}$
- If we work soley in $\mathbb{R}^n$ then there is another way to say this:
    - The first column of the matrix is the output of $\begin{bmatrix}1\\0\end{bmatrix}$ under the linear transform
    - The second column of the matrix is the output of $\begin{bmatrix}0\\1\end{bmatrix}$ under the linear transform

- Composition of linear transforms corresponds to matrix multiplication
    - I won't go through all the details, but
    - If you have two linear transforms, both of which have the same domain and codomain, then you may choose a basis and get a matrix for each of them
    - You may compose them, you get another linear transform
        - Hence, the composition has a matrix too
    - If you work through the details of this, it turns out that the matrix of the composition corresponds to the matrix multiplication of the matrices of the two transforms
    - Example:
        - Suppose we want to compose the linear transform whose matrix is $\begin{bmatrix} 2 & 1 \\ 3 & 2 \end{bmatrix}$ with $\begin{bmatrix} 3 & 7 \\ 1 & 1 \end{bmatrix}$
        - Well, we know that $\begin{bmatrix} 1\\0\end{bmatrix}$ goes to $\begin{bmatrix} 2 \\ 3 \end{bmatrix}$
        - And we know that $\begin{bmatrix} 2 \\ 3 \end{bmatrix}$ goes to $2\begin{bmatrix} 3\\1 \end{bmatrix}+3\begin{bmatrix} 7 \\ 1 \end{bmatrix}$
        - Hence the top left entry is $2*3+3*7$
            - But this is the same as what you get when you matrix multiply the above two matrices

- To be continued:
  - Further topics
    - Change of basis formula
    - Gauss-Jordan
    - Inner products, orthonormality
    - Gram-Schimdt
      - Interpolation formula for quaternions
    - The first isomorphism theorem for inner product spaces
    - Eigenvalues
    - Determinants
    - Duality
    - Tensors
  - Further material
    - [Essense of linear algebra](https://www.youtube.com/playlist?list=PLZHQObOWTQDPD3MizzM2xVFitgF8hE_ab) by 3blue1brown
    - Linear algebra done right by Axler
    - http://number-none.com/product/Understanding%20Slerp,%20Then%20Not%20Using%20It/


# Chapter 2: Multi-armed Bandits
Neel Shah, 2021
**Solutions are not guaranteed to be correct**

**Exercise 2.1 In $\epsilon$-greedy action selection, for the case of two actions and $\epsilon$ = 0.5, what is the probability that the greedy action is selected?**
.5 * 1 (exploit case) + .5 * .5 (explore case) = .75

**Exercise 2.2: Bandit example Consider a k-armed bandit problem with k = 4 actions, denoted 1, 2, 3, and 4. Consider applying to this problem a bandit algorithm using "-greedy action selection, sample-average action-value estimates, and initial estimates of Q1(a) = 0, for all a. Suppose the initial sequence of actions and rewards is A1 = 1, R1 =-1,A2 =2,R2 =1,A3 =2,R3 =-2,A4 =2,R4 =2,A5 =3,R5 =0. Onsome of these time steps the " case may have occurred, causing an action to be selected at random. On which time steps did this definitely occur? On which time steps could this possibly have occurred?**
Definitely:
4 - Q(2) is < 0 while Q(3), Q(4) = 0 so choosing A=2 must be a random choice (after Q(2)=1)
5 - Q(3) = 0 but Q(2) > 0, so choosing action 3 must be random
Possible:
1 - all Q_1's = 0 which means that R1 can either be explore or picking one such action (after Q(1) = -1)
2 - Q_1(1) = -1, rest are 0, so this choice could be random or picking one such Q(2) = 0
3 - now, Q(2) = 1, so choosing 2 can be random choice or exploitation (after Q(2) = -.5)

**Exercise 2.3 In the comparison shown in Figure 2.2, which method will perform best in the long run in terms of cumulative reward and probability of selecting the best action? How much better will it be? Express your answer quantitatively.**
$$
\epsilon=.01
$$
As $t\rightarrow\inf$, the probability of selecting the optimal action approaches $1-\epsilon=.99$

Normally, $$Q_n = \frac{R_1 + ... + R_{n-1}}{n-1}$$ However, each additional reward would require additionally memory to store it and computation to compute the sum.
Instead, we can devise a way to incrementally update.  Given $Q_n$ and the $n$th reward $R_n$,
$$
\begin{align*}
Q_{n+1} &= \frac{1}{n}\sum_{i=1}^{n}R_i  \\
&= \frac{1}{n}\left(R_n + \sum{i=1}_{n-1}R_i\right)  \\
&= \frac{1}{n}\left(R_n + (n-1)\frac{1}{n-1}\sum{i=1}_{n-1}R_i\right)  \\
&= \frac{1}{n}\big(R_n + (n-1)Q_n\big)  \\
&= \frac{1}{n}\big(R_n + nQ_n - Q_n\big)  \\
&= Q_n + \frac{1}{n}[R_n - Q_n]
\end{align*}
$$
This implementation requires memory only for $Q_n$ and $n$ and slight computation for each new reward. $$ NewEstimate \leftarrow OldEstimate + StepSize [Target - OldEstimate] $$

**Exercise 2.4 If the step-size parameters, $\alpha_n$, are not constant, then the estimate $Q_n$ is a weighted average of previously received rewards with a weighting different from that given by (2.6). What is the weighting on each prior reward for the general case, analogous to (2.6), in terms of the sequence of step-size parameters?**
$$
\begin{align}
Q_{n+1} &= Q_n + \alpha_n\left[R_n - Q_n\right] \\
&= \alpha_nR_n + \left(1 - \alpha_n\right)Q_n \\
&= \alpha_nR_n + (1-\alpha_n)\left[\alpha_{n-1}R_{n-1} + (1-\alpha_{n-1}Q_{n-1}\right] \\
&= \alpha_nR_n + (1-\alpha_n)\alpha_{n-1}R_{n-1} + (1-\alpha_n)(1-\alpha_{n-1}Q_{n-1})\\
&= ... \\
&= Q_1\prod_{i=1}^{n}(1-\alpha_i) + \sum_{i=0}^{n}\left(R_{n-i}\alpha_{n-i}\prod_{j=0}^{i-1}\left(1-\alpha_{n-j}\right)\right)
\end{align}
$$

**Exercise 2.5 (programming) Design and conduct an experiment to demonstrate the difficulties that sample-average methods have for nonstationary problems. Use a modified version of the 10-armed testbed in which all the $q_{\star}(a)$ start out equal and then take independent random walks (say by adding a normally distributed increment with mean 0 and standard deviation 0.01 to all the $q_{\star}(a)$ on each step). Prepare plots like Figure 2.2 for an action-value method using sample averages, incrementally computed, and another action-value method using a constant step-size parameter, $\alpha = 0.1$. Use $\epsilon = 0.1$ and longer runs, say of 10,000 steps.**.

```python
import numpy as np
from typing import List, Callable
import matplotlib.pyplot as plt
#10-armed bandit

#expected reward for action a
q_star = [0 for _ in range(10)]

def sample_average_bandit(steps: int,
                          eps: float,
                          rewards: Callable,
                          num_actions: int = 10,
                          analyze: Callable = None, q=None):
    """Sample average bandit"""
    #initial guess of rewards
    q = q or [0 for _ in range(num_actions)]
    n = [0 for _ in range(num_actions)]
    for step in range(1,steps+1):
        if np.random.random() < eps:
            action = np.random.randint(0, num_actions)
        else:
            action = np.argmax(q)
        reward = rewards(action)
        n[action] += 1
        q[action] = q[action] + (1/n[action]) * (reward - q[action])
        if analyze is not None:
            analyze(step, eps, action, reward)

def constant_step_bandit(steps: int,
                         eps: float,
                         rewards: Callable,
                         step_size: float,
                         num_actions: int = 10,
                         analyze: Callable = None, q=None):
    """Constant step bandit"""
    #initial guess of rewards
    q = q or [0 for _ in range(num_actions)]
    for step in range(1,steps+1):
        if np.random.random() < eps:
            action = np.random.randint(0, num_actions)
        else:
            action = np.argmax(q)
        reward = rewards(action)
        q[action] = q[action] + step_size * (reward - q[action])
        if analyze is not None:
            analyze(step, eps, action, reward)

def gen_funcs(q_star: List[float]):
    q = q_star.copy()
    optimal = -1
    def rewards(action: int):
        nonlocal q, optimal
        optimal = np.argmax(q)
        reward = q[action]
        q = q + np.random.normal(0, .01, 10)
        return reward

    class analyze:
        def __init__(self):
            self.step = 1
            self.rewards = 0
            self.average_reward = []
            self.optimal_actions = 0
            self.optimal_action_perc = []

        def __call__(self, step, epsilon, action, reward):
            self.step = step
            self.rewards += reward
            self.average_reward.append(self.rewards/self.step)
            self.optimal_actions += 1 if action == optimal else 0
            self.optimal_action_perc.append(self.optimal_actions/self.step)

        def plot(self, label):
            fig, ax = plt.subplots()
            ax.plot(np.arange(self.step), self.average_reward)
            ax.set(xlabel='steps', ylabel='Average reward', title=label)
            plt.show()

            fig, ax = plt.subplots()
            ax.plot(np.arange(self.step), self.optimal_action_perc)
            ax.set(xlabel='steps', ylabel='Optimal action %', title=label)
            plt.show()

    return rewards, analyze

rewards, analyze = gen_funcs(q_star)
analyze_obj = analyze()
sample_average_bandit(10000, .1, rewards, 10, analyze_obj)
analyze_obj.plot('Sample Average')

rewards, analyze = gen_funcs(q_star)
analyze_obj = analyze()
constant_step_bandit(10000, .1, rewards, .1, 10, analyze_obj)
analyze_obj.plot('Constant step-size')
```

**Exercise 2.6: Mysterious Spikes The results shown in Figure 2.3 should be quite reliable because they are averages over 2000 individual, randomly chosen 10-armed bandit tasks. Why, then, are there oscillations and spikes in the early part of the curve for the optimistic method? In other words, what might make this method perform particularly better or worse, on average, on particular early steps?**
  $\epsilon = 0$ implies that will always choose the action that it thinks is optional, with ties broken randomly. With all $Q_1(a) = 5 \forall a$ the top choice initially has a $1/10$ probability of being chosen and so-forth once $Q_i(a)$ is updated.
**Exercise 2.7: Unbiased Constant-Step-Size Trick In most of this chapter we have used sample averages to estimate action values because sample averages do not produce the initial bias that constant step sizes do (see the analysis leading to (2.6)). However, sample averages are not a completely satisfactory solution because they may perform poorly on nonstationary problems. Is it possible to avoid the bias of constant step sizes while retaining their advantages on nonstationary problems? One way is to use a step size of
$$\beta_{n}=\alpha/\bar{o}_n$$, (2.8) to process the nth reward for a particular action, where $\alpha>0$ is a conventional constant
step size, and $\bar{o}_n$ is a trace of one that starts at 0:
$$\bar{o}_n=\bar{o}_{n-1}+\alpha(1-\bar{o}_{n-1})$$, for $n\geq0$, with $\bar{o}_0=0$ (2.9)
Carry out an analysis like that in (2.6) to show that Qn is an exponential recency-weighted average without initial bias.**
$$
\begin{align*}
\bar{o}_n &= \bar{o}_{n-1} + \alpha(1-\bar{o}_{n-1}) \\
&= \bar{o}_{n-1} + \alpha - \alpha\bar{o}_{n-1} \\
&= \alpha + (1-\alpha)(\alpha + (1-\alpha)\bar{o}_{n-2}) \\
&= \alpha + (1-\alpha)\alpha + (1-\alpha)^2(\alpha + (1-\alpha)\bar{o}_{n-3}) \\
&= \sum_{i=0}^{n}(1-\alpha)^i\alpha \\
&= \alpha\left(\frac{1-(1-\alpha)^{n+1}}{\alpha}\right) \\
&= 1 - (1-\alpha)^{n+1}
\end{align*}
$$
Because $\bar{o}_n$ is a geometric series which converges at 1 (not shown), the bandit has initial bias to promote exploration, but the step-size $\beta_n$ decreases, eventually converging to a constant step-size $\alpha$

**Exercise 2.8: UCB Spikes In Figure 2.4 the UCB algorithm shows a distinct spike in performance on the 11th step. Why is this? Note that for your answer to be fully satisfactory it must explain both why the reward increases on the 11th step and why it decreases on the subsequent steps. Hint: If c = 1, then the spike is less prominent.**
$N_t(a)=0\space \forall a$ until step 11. At step 11, because all $N_t(a)$ are equal (at that point), the algorithm chooses the action that it predicts the highest reward. For subsequent steps, $N_t(a) > 1$ so the confidence level term decreases causing more actions to be explored and reward estimates to be updated.

**Exercise 2.9 Show that in the case of two actions, the soft-max distribution is the same as that given by the logistic, or sigmoid, function often used in statistics and artificial neural networks.**
*TODO*

**Exercise 2.10 Suppose you face a 2-armed bandit task whose true action values change randomly from time step to time step. Specifically, suppose that, for any time step, the true values of actions 1 and 2 are respectively 10 and 20 with probability 0.5 (case A), and 90 and 80 with probability 0.5 (case B). If you are not able to tell which case you face at any step, what is the best expected reward you can achieve and how should you behave to achieve it? Now suppose that on each step you are told whether you are facing case A or case B (although you still don’t know the true action values). This is an associative search task. What is the best expected reward you can achieve in this task, and how should you behave to achieve it?**
Regular bandit:
max expected reward $.5*10+.5*90 = 50 = .5*20+.5*80$, you can achieve this only by assuming that all cases are consistent (don't switch between guessing case A and B) and choosing the same action
Contextual bandit:
max expected reward 20 in case A and 90 in case B. You can learn the rewards for actions for each tasks independently.

**Exercise 2.11 (programming) Make a figure analogous to Figure 2.6 for the nonstationary case outlined in Exercise 2.5. Include the constant-step-size "-greedy algorithm with ↵=0.1. Use runs of 200,000 steps and, as a performance measure for each algorithm and parameter setting, use the average reward over the last 100,000 steps.**
*Not fully implemented below*

```python
from scipy.special import softmax

eps = [1/128, 1/64, 1/32, 1/16, 1/8, 1/4]
alphas = [1/32,1/16,1/8,1/4,1/2,1,2]
cs = [1/16,1/8,1/4,1/2,1,2,4]
q0s = [1/4,1/2,1,2,4]

def gen_rewards(q_star: List[float]):
    q = q_star.copy()
    optimal = -1
    def rewards(action: int):
        nonlocal q, optimal
        optimal = np.argmax(q)
        reward = q[action]
        q = q + np.random.normal(0, .01, 10)
        return reward
    return rewards

class analyze:
    d = {}
    def __init__(self, alg):
        self.alg = alg
        if alg not in analyze.d:
            analyze.d[alg] = {}
        self.sum = 0

    def __call__(self, *args):
        if len(args) == 3:
            self._call(args[0], args[1], args[2])
        elif len(args) == 4:
            self._call_eps(args[0], args[1], args[2], args[3])
    def _call(self, step, param, reward):
        if step > 100000:
            self.sum += reward
            if step == 200000:
                analyze.d[self.alg][param] = self.sum/100000

    def _call_eps(self, step, eps, action, reward):
        """call for greedy algs"""
        if step > 100000:
            self.sum += reward
            if step == 200000:
                analyze.d[self.alg][eps] = self.sum/100000
    def plot():
        fig, ax = plt.subplots()
        for alg in analyze.d:
            ax.plot(analyze.d[alg].keys(), analyze.d[alg].values(), label=alg)
        plt.show()

def ucb(steps: int,
        c: float,
        rewards: Callable,
        num_actions: int = 10,
        analyze: Callable = None,
        q=None):
    """UCB Action selection"""
    #initial guess of rewards
    q = q or [0 for _ in range(num_actions)]
    n = [0 for _ in range(num_actions)]
    for step in range(1,steps+1):
        action = np.argmax(q + c*np.sqrt(np.log(step)/n))
        reward = rewards(action)
        n[action] += 1
        q[action] = q[action] + (1/n[action]) * (reward - q[action])
        if analyze is not None:
            analyze(step, c, reward)

def gradient_bandit(steps: int,
                    rewards: Callable,
                    step_size: float,
                    num_actions: int = 10,
                    analyze: Callable = None,
                    q=None):
    """Gradient bandit"""
    #initial guess of rewards
    q = q or [0 for _ in range(num_actions)]
    H = [0 for _ in range(num_actions)] #preference parameter
    R = 0
    for step in range(1,steps+1):
        probs = softmax(H)
        action = np.random.choice(num_actions, p=probs)
        reward = rewards(action)
        for a in range(num_actions):
            if a == action:
                H[a] = H[a] + step_size*(reward - R)*(1-probs[a])
            else:
                H[a] = H[a] - step_size*(reward - R)*probs[a]
            R = R + (1/step)*(reward - R)
        if analyze is not None:
            analyze(step, step_size, reward)

for e in eps:
    rewards = gen_rewards(q_star)
    analyze_obj = analyze('eps-greedy')
    constant_step_bandit(200000, e, rewards, .1, 10, analyze_obj)

for a in alphas:
    rewards = gen_rewards(q_star)
    analyze_obj = analyze('gradient bandit')
    gradient_bandit(200000, rewards, a, 10, analyze_obj)

for c in cs:
    rewards = gen_rewards(q_star)
    analyze_obj = analyze('UCB')
    ucb(200000, c, rewards, 10, analyze_obj)

for q0 in q0s:
    rewards = gen_rewards(q_star)
    analyze_obj = analyze('greedy with optimistic initialization alpha=.1')
    constant_step_bandit(200000, 0, rewards, .1, 10, analyze_obj, [q0]*10)

analyze.plot()
```

```python

```


# 3 Degrees Of Freedome (DOF)

```python
%matplotlib inline
import sympy as sym
import sympy.physics.mechanics as me
from sympy import init_printing
init_printing(use_latex='mathjax')

from pydy.system import System
import numpy as np
import matplotlib.pyplot as plt

```

```python
x0,y0,psi,x,y,beta,x,y,u,v,r,V = me.dynamicsymbols(r'x0 y0 \psi x y \beta x y u v r V')
m,Izz,t = sym.symbols(r'm Izz t')
```

```python
earth_frame = me.ReferenceFrame(name = 'E')
ship_frame = me.ReferenceFrame(name = 'S')
ship_frame.orient(parent = earth_frame,rot_type='Axis',amounts = (psi,earth_frame.z))
#ship_frame.set_ang_vel(otherframe=earth_frame,value = r*earth_frame.z)
```

```python
ship_frame.dcm(otherframe=earth_frame)
```

$$\left[\begin{matrix}\cos{\left (\psi{\left (t \right )} \right )} & \sin{\left (\psi{\left (t \right )} \right )} & 0\\- \sin{\left (\psi{\left (t \right )} \right )} & \cos{\left (\psi{\left (t \right )} \right )} & 0\\0 & 0 & 1\end{matrix}\right]$$

```python
O0 = me.Point('O0')
O0.set_vel(frame = earth_frame,value = 0)

O = me.Point('O')
#O.set_pos(otherpoint=O0,value = x0*earth_frame.x + y0*earth_frame.y)
O.set_pos(otherpoint=O0,value = x*ship_frame.x + y*ship_frame.y)

O.set_vel(frame = ship_frame,value = u*ship_frame.x + v*ship_frame.y)

O.v1pt_theory(otherpoint=O0,outframe=earth_frame,interframe=ship_frame)
```

$$(u - y \dot{\psi})\mathbf{\hat{s}_x} + (v + x \dot{\psi})\mathbf{\hat{s}_y}$$

```python
O.pos_from(O0)
```

$$x\mathbf{\hat{s}_x} + y\mathbf{\hat{s}_y}$$

```python
O.vel(earth_frame).express(earth_frame)
```

$$(\left(u - y \dot{\psi}\right) \operatorname{cos}\left(\psi\right) - \left(v + x \dot{\psi}\right) \operatorname{sin}\left(\psi\right))\mathbf{\hat{e}_x} + (\left(u - y \dot{\psi}\right) \operatorname{sin}\left(\psi\right) + \left(v + x \dot{\psi}\right) \operatorname{cos}\left(\psi\right))\mathbf{\hat{e}_y}$$

```python
O.acc(earth_frame).express(earth_frame)
```

$$(- \left(\left(u - y \dot{\psi}\right) \dot{\psi} + x \ddot{\psi} + \dot{\psi} \dot{x} + \dot{v}\right) \operatorname{sin}\left(\psi\right) + \left(- \left(v + x \dot{\psi}\right) \dot{\psi} - y \ddot{\psi} - \dot{\psi} \dot{y} + \dot{u}\right) \operatorname{cos}\left(\psi\right))\mathbf{\hat{e}_x} + (\left(\left(u - y \dot{\psi}\right) \dot{\psi} + x \ddot{\psi} + \dot{\psi} \dot{x} + \dot{v}\right) \operatorname{cos}\left(\psi\right) + \left(- \left(v + x \dot{\psi}\right) \dot{\psi} - y \ddot{\psi} - \dot{\psi} \dot{y} + \dot{u}\right) \operatorname{sin}\left(\psi\right))\mathbf{\hat{e}_y}$$

```python
kinematical_differential_equations = [u - x.diff(),
                                      v - y.diff(),
                                      r - psi.diff()]
coordinates = [x,y,psi,]
speeds = [u,v,r,]
```

```python
kane = me.KanesMethod(earth_frame,coordinates, speeds,kinematical_differential_equations)
```

```python
forces = 1*ship_frame.x
mass = me.Particle('m', O, m)
```

```python
inertia_dyadic = me.inertia(ship_frame, ixx = 0, iyy = 0, izz = Izz)
inertia_dyadic
```

$$Izz\mathbf{\hat{s}_z}\otimes \mathbf{\hat{s}_z}$$

```python
inertia = (inertia_dyadic, O)
```

```python
ship = me.RigidBody('ship',masscenter=O,frame = ship_frame,mass = mass,inertia=inertia)
```

```python
fr, frstar = kane.kanes_equations(bodies = [ship],loads = [(O, forces)])
```

```python
M = kane.mass_matrix_full
M
```

$$\left[\begin{matrix}1 & 0 & 0 & 0 & 0 & 0\\0 & 1 & 0 & 0 & 0 & 0\\0 & 0 & 1 & 0 & 0 & 0\\0 & 0 & 0 & m & 0 & - m y{\left (t \right )}\\0 & 0 & 0 & 0 & m & m x{\left (t \right )}\\0 & 0 & 0 & - m y{\left (t \right )} & m x{\left (t \right )} & Izz + m \left(x^{2}{\left (t \right )} + y^{2}{\left (t \right )}\right)\end{matrix}\right]$$

```python
f = kane.forcing_full
f
```

$$\left[\begin{matrix}u{\left (t \right )}\\v{\left (t \right )}\\r{\left (t \right )}\\- m \left(- \left(r{\left (t \right )} x{\left (t \right )} + v{\left (t \right )}\right) r{\left (t \right )} - r{\left (t \right )} v{\left (t \right )}\right) + 1\\- m \left(\left(- r{\left (t \right )} y{\left (t \right )} + u{\left (t \right )}\right) r{\left (t \right )} + r{\left (t \right )} u{\left (t \right )}\right)\\m \left(- \left(r{\left (t \right )} x{\left (t \right )} + v{\left (t \right )}\right) r{\left (t \right )} - r{\left (t \right )} v{\left (t \right )}\right) y{\left (t \right )} - m \left(\left(- r{\left (t \right )} y{\left (t \right )} + u{\left (t \right )}\right) r{\left (t \right )} + r{\left (t \right )} u{\left (t \right )}\right) x{\left (t \right )} - y{\left (t \right )}\end{matrix}\right]$$

```python
(M.inv() * f)
```

$$\left[\begin{matrix}u{\left (t \right )}\\v{\left (t \right )}\\r{\left (t \right )}\\\frac{m^{3} \left(\left(- r{\left (t \right )} y{\left (t \right )} + u{\left (t \right )}\right) r{\left (t \right )} + r{\left (t \right )} u{\left (t \right )}\right) x{\left (t \right )} y{\left (t \right )}}{- m^{3} x^{2}{\left (t \right )} + m \left(- m^{2} y^{2}{\left (t \right )} + m \left(Izz + m \left(x^{2}{\left (t \right )} + y^{2}{\left (t \right )}\right)\right)\right)} + \frac{m^{2} \left(m \left(- \left(r{\left (t \right )} x{\left (t \right )} + v{\left (t \right )}\right) r{\left (t \right )} - r{\left (t \right )} v{\left (t \right )}\right) y{\left (t \right )} - m \left(\left(- r{\left (t \right )} y{\left (t \right )} + u{\left (t \right )}\right) r{\left (t \right )} + r{\left (t \right )} u{\left (t \right )}\right) x{\left (t \right )} - y{\left (t \right )}\right) y{\left (t \right )}}{- m^{3} x^{2}{\left (t \right )} + m \left(- m^{2} y^{2}{\left (t \right )} + m \left(Izz + m \left(x^{2}{\left (t \right )} + y^{2}{\left (t \right )}\right)\right)\right)} + \frac{\left(- m \left(- \left(r{\left (t \right )} x{\left (t \right )} + v{\left (t \right )}\right) r{\left (t \right )} - r{\left (t \right )} v{\left (t \right )}\right) + 1\right) \left(- m^{3} x^{2}{\left (t \right )} + m^{3} y^{2}{\left (t \right )} + m \left(- m^{2} y^{2}{\left (t \right )} + m \left(Izz + m \left(x^{2}{\left (t \right )} + y^{2}{\left (t \right )}\right)\right)\right)\right)}{m \left(- m^{3} x^{2}{\left (t \right )} + m \left(- m^{2} y^{2}{\left (t \right )} + m \left(Izz + m \left(x^{2}{\left (t \right )} + y^{2}{\left (t \right )}\right)\right)\right)\right)}\\- \frac{m^{2} \left(- m \left(- \left(r{\left (t \right )} x{\left (t \right )} + v{\left (t \right )}\right) r{\left (t \right )} - r{\left (t \right )} v{\left (t \right )}\right) + 1\right) x{\left (t \right )} y{\left (t \right )}}{- m^{3} x^{2}{\left (t \right )} + m \left(- m^{2} y^{2}{\left (t \right )} + m \left(Izz + m \left(x^{2}{\left (t \right )} + y^{2}{\left (t \right )}\right)\right)\right)} - \frac{m^{2} \left(m \left(- \left(r{\left (t \right )} x{\left (t \right )} + v{\left (t \right )}\right) r{\left (t \right )} - r{\left (t \right )} v{\left (t \right )}\right) y{\left (t \right )} - m \left(\left(- r{\left (t \right )} y{\left (t \right )} + u{\left (t \right )}\right) r{\left (t \right )} + r{\left (t \right )} u{\left (t \right )}\right) x{\left (t \right )} - y{\left (t \right )}\right) x{\left (t \right )}}{- m^{3} x^{2}{\left (t \right )} + m \left(- m^{2} y^{2}{\left (t \right )} + m \left(Izz + m \left(x^{2}{\left (t \right )} + y^{2}{\left (t \right )}\right)\right)\right)} - \frac{m \left(- m^{2} y^{2}{\left (t \right )} + m \left(Izz + m \left(x^{2}{\left (t \right )} + y^{2}{\left (t \right )}\right)\right)\right) \left(\left(- r{\left (t \right )} y{\left (t \right )} + u{\left (t \right )}\right) r{\left (t \right )} + r{\left (t \right )} u{\left (t \right )}\right)}{- m^{3} x^{2}{\left (t \right )} + m \left(- m^{2} y^{2}{\left (t \right )} + m \left(Izz + m \left(x^{2}{\left (t \right )} + y^{2}{\left (t \right )}\right)\right)\right)}\\\frac{m^{3} \left(\left(- r{\left (t \right )} y{\left (t \right )} + u{\left (t \right )}\right) r{\left (t \right )} + r{\left (t \right )} u{\left (t \right )}\right) x{\left (t \right )}}{- m^{3} x^{2}{\left (t \right )} + m \left(- m^{2} y^{2}{\left (t \right )} + m \left(Izz + m \left(x^{2}{\left (t \right )} + y^{2}{\left (t \right )}\right)\right)\right)} + \frac{m^{2} \left(- m \left(- \left(r{\left (t \right )} x{\left (t \right )} + v{\left (t \right )}\right) r{\left (t \right )} - r{\left (t \right )} v{\left (t \right )}\right) + 1\right) y{\left (t \right )}}{- m^{3} x^{2}{\left (t \right )} + m \left(- m^{2} y^{2}{\left (t \right )} + m \left(Izz + m \left(x^{2}{\left (t \right )} + y^{2}{\left (t \right )}\right)\right)\right)} + \frac{m^{2} \left(m \left(- \left(r{\left (t \right )} x{\left (t \right )} + v{\left (t \right )}\right) r{\left (t \right )} - r{\left (t \right )} v{\left (t \right )}\right) y{\left (t \right )} - m \left(\left(- r{\left (t \right )} y{\left (t \right )} + u{\left (t \right )}\right) r{\left (t \right )} + r{\left (t \right )} u{\left (t \right )}\right) x{\left (t \right )} - y{\left (t \right )}\right)}{- m^{3} x^{2}{\left (t \right )} + m \left(- m^{2} y^{2}{\left (t \right )} + m \left(Izz + m \left(x^{2}{\left (t \right )} + y^{2}{\left (t \right )}\right)\right)\right)}\end{matrix}\right]$$

## Create a PyDy system

```python
constants = {
    m:1,
    Izz:1,
}

initial_conditions = {
    x:0,
    y:0,
    psi:0,
    u:0,
    v:0,
    r:0,

}

times = np.linspace(0,10,100)

sys = System(kane,constants=constants,initial_conditions=initial_conditions,times = times)
```

```python
result = sys.integrate()
```

```python
fig,ax = plt.subplots()
ax.plot(times,result[:,0])
ax.set_xlabel('time [s]')
ax.set_ylabel('x [m]')
```

```python
p = O.pos_from(O0).express(earth_frame)
x0_ = p.dot(ship_frame.x)
x0_
```

$$\left(x{\left (t \right )} \sin{\left (\psi{\left (t \right )} \right )} + y{\left (t \right )} \cos{\left (\psi{\left (t \right )} \right )}\right) \sin{\left (\psi{\left (t \right )} \right )} + \left(x{\left (t \right )} \cos{\left (\psi{\left (t \right )} \right )} - y{\left (t \right )} \sin{\left (\psi{\left (t \right )} \right )}\right) \cos{\left (\psi{\left (t \right )} \right )}$$

```python
y0_ = p.dot(ship_frame.y)
y0_
```

$$\left(x{\left (t \right )} \sin{\left (\psi{\left (t \right )} \right )} + y{\left (t \right )} \cos{\left (\psi{\left (t \right )} \right )}\right) \cos{\left (\psi{\left (t \right )} \right )} - \left(x{\left (t \right )} \cos{\left (\psi{\left (t \right )} \right )} - y{\left (t \right )} \sin{\left (\psi{\left (t \right )} \right )}\right) \sin{\left (\psi{\left (t \right )} \right )}$$

```python
f_x0 = sym.lambdify((x,y,psi,t),x0_,"numpy")
f_y0 = sym.lambdify((x,y,psi,t),y0_,"numpy")

```

```python
x0_sim = f_x0(result[:,0],result[:,1],result[:,2],times)
```

```python
fig,ax = plt.subplots()
ax.plot(times,x0_sim)
ax.set_xlabel('time [s]')
ax.set_ylabel('x0 [m]')
```

```python
y0_sim = f_y0(result[:,0],result[:,1],result[:,2],times)
```

```python
fig,ax = plt.subplots()
ax.plot(times,y0_sim)
ax.set_xlabel('time [s]')
ax.set_ylabel('y0 [m]')
```

```python
fig,ax = plt.subplots()
ax.plot(y0_sim,x0_sim)
ax.set_xlabel('y0 [m]')
ax.set_ylabel('x0 [m]')

```

```python

```