```python
%matplotlib inline
import matplotlib.pyplot as plt
import numpy as np
import sympy as sym
```

```python
t = sym.symbols('t', real=True)
k = sym.symbols('k')

w_0 = 5
w_s = 50
x = sym.cos(w_0 * t)
```

```python
s = sym.sequence(x.subs(t, k * 2 * sym.pi / w_s), (k, 0, 100))
s
```

    SeqFormula(cos(pi*k/5), (k, 0, 100))

```python
a = np.fromiter(s, float)
```

```python
plt.plot(a);
```

```python
s.formula
```

    cos(pi*k/5)

```python
s.interval
```

    [0, 100]

```python
s.length
```

    101

```python
s.start
```

    0

```python
s.stop
```

    100

```python
s.variables
```

    (k,)

```python
s.free_symbols
```

    set()

```python
s.coeff_mul(42)
```

    SeqFormula(42*cos(pi*k/5), (k, 0, 100))

```python
s.coeff(1)
```

    1/4 + sqrt(5)/4

```python
s[1]
```

    1/4 + sqrt(5)/4

```python
s[:10]
```

    [1,
     1/4 + sqrt(5)/4,
     -1/4 + sqrt(5)/4,
     -sqrt(5)/4 + 1/4,
     -sqrt(5)/4 - 1/4,
     -1,
     -sqrt(5)/4 - 1/4,
     -sqrt(5)/4 + 1/4,
     -1/4 + sqrt(5)/4,
     1/4 + sqrt(5)/4]

```python
def sinc(x):
    return sym.sin(x) / x
```

```python
T = 2 * sym.pi / w_s

y = sum(s[n] * sinc(sym.pi / T * (t - n * T))
        for n in range(s.start, s.stop + 1))
```

```python
sym.plot(y, (t, 0, 5), xlabel='$t$', ylabel='$y(t)$', ylim=(-1.2, 1.2));
```

```python
random = sum(np.random.normal(scale=0.5) * sinc(sym.pi / T * (t - n * T))
             for n in range(s.start, s.stop + 1))
```

```python
sym.plot(random, (t, 0, 5), xlabel='$t$', ylabel='$y(t)$', ylim=(-1.2, 1.2));
```


```python
%matplotlib inline
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.patches as patches
from matplotlib import cm
from mpl_toolkits.mplot3d import Axes3D
plt.rcParams['font.size'] = 13
plt.rcParams['axes.spines.right'] = False
plt.rcParams['ytick.right'] = False
plt.rcParams['axes.spines.top'] = False
plt.rcParams['xtick.top'] = False
```

### What kind of stimulus causes a cell to fire?

```python
# Generate an example filter
dt = 0.01
freq = 8
tau = 0.05
t = np.arange(-0.25, 0+dt, dt)
f = -np.sin(2*np.pi*freq*t)*np.exp(t/tau)
f = f / np.linalg.norm(f)
#plt.plot(t, f, 'k-')
```

```python
np.random.seed(13)
l = 1000;      # stimulus length in seconds
nPlot = 100; # samples to plot
winSize = 0.25 # filter size (s)
shift = 25     # shift to avoid negative indices

# Simulate data
time = np.arange(0, l/dt+dt, dt)                # time stamps
s = 0.1*np.random.randn(int(l/dt));             # white noise stimulus
z = np.convolve(s, np.flip(f, axis=0), 'full')  # 1. convolve with the flipped filter
mu = z * (z>0)                                  # 2. rectifying non-linearity
spikes = np.random.poisson(mu)                  # 3. spike generation

# plot the stimulus, spikes, and the window containing the stimulus preceeding each spike
plt.figure(figsize=(15, 3))
ah = plt.subplot(1, 2, 1)
plt.plot(time[shift:nPlot+shift], s[shift:nPlot+shift], 'k-')
ah.set_ylabel('Intensity')
ah.set_xlabel('Time (s)')
ah.set_title('Stimulus and spikes')
ah.set_ylim([-0.4, 0.6])
plt.plot(time[shift:nPlot+shift], 0.5*np.ones(nPlot), '-', color='gray')
for idx in np.argwhere(spikes[shift:nPlot+shift]>0):
    plt.plot([time[idx+shift], time[idx+shift]], [0.45, 0.55], '-', color='gray')
    rect = patches.Rectangle([time[idx+shift]-winSize, -0.3], winSize, 0.6, linewidth=2,edgecolor='gray',facecolor='none')
    ah.add_patch(rect)

# Calculate and plot the spike-triggered average (STA)
ah = plt.subplot(1, 2, 2)
nSta = int(winSize/dt)+1;
sta = np.zeros(nSta)
for idx in np.argwhere(spikes[shift:nPlot+shift]>0):
    sta += spikes[shift+idx]*s[idx[0]+shift-nSta+1:idx[0]+shift+1]
staNorm = sta / np.linalg.norm(sta)
plt.plot(np.arange(-winSize, 0+dt, dt), staNorm, 'ko-')
#plt.plot(t, f, '-', color='gray')
ah.set_ylabel('STA magnitude')
ah.set_xlabel('Time (s)')
ah.set_title('Spike-triggered average (STA)');
```

The STA is the average stimulus that preceded a spike, and consequently, the x-axis has negative time values with time zero (the occurrence of a spike) located in the right most part of the plot. The STA thus tells us what the average stimulus looks like that preceded a spike, and it should therefore not be confused with peri-stimulus time histograms (PSTHs) which represent cell responses to a fixed stimulus.

### What about the LN model?

The linear-nonlinear (LN) model assumes that a cell's behavior, its stimulus-response mapping, can be described using a cascade of a linear operation and a nonlinear function. The linear operation, an inner product between a filter ($\mathbf{w}$) and the stimulus ($\mathbf{x}$), results in a similarity score ($z$) that is transformed into a mean firing rate ($y$) by the nonlinear function ($f$). Mathematically, this is expressed as:

\begin{equation}
 y = f(\mathbf{w}^T\mathbf{x}) = f \left( \sum_j w_j x_j \right).
\end{equation}

If needed, it is also possible to generate realizations of example spike trains by sampling from a spike event distribution, usually a Poisson distribution, with the mean rate $y$. In the end, it is then possible to describe the full LN model as a three step procedure:
1. Calculate the inner product between the filter and each stimulus, or equivalently to filter the stimulus sequence using the filter.
2. Pass the resulting similarity score at each time point through the nonlinearity so as to obtain a firing rate.
3. Draw samples from the event rate distribution to get a realization of a spike train.

These three steps are illustrated below using an example with an oscillating stimulus.

```python
tMax = 0.75;
time = np.arange(0, tMax+dt, dt)
tIdx = 31
nonlin = lambda z: z * (z>0)
#nonlin = lambda(z): (z/2)**2
s = np.sin(2*np.pi*freq*time)                             # generate an oscillating stimulus
z = np.convolve(s, np.flip(f, axis=0), 'full')[0:s.size]  # 1. filter the stimulus sequence using the LN model's filter
y = nonlin(z)                                             # 2. apply the nonlinearity
spikeCounts = np.random.poisson(y)                        # 3. sample spike counts

fig = plt.figure(figsize=(15, 10))
ah = plt.subplot(4, 2, 1)
plt.plot(time, s, 'k.-')
rect = patches.Rectangle([time[tIdx]-winSize, -1.1], winSize, 2.2, linewidth=2,edgecolor='b',facecolor='none')
ah.add_patch(rect)
rect = patches.Rectangle([time[tIdx+31]-winSize, -1.1], winSize, 2.2, linewidth=2,edgecolor='r',facecolor='none')
ah.add_patch(rect)
ah.set_xticklabels([])
ah.set_ylabel('Intensity')
ah.set_title('Stimulus sequence')

ah = plt.subplot(4, 2, 2)
plt.plot(t, f, 'ko-')
ah.set_ylabel('Filter magnitude')
ah.set_xlabel('Time (s)')
ah.set_title('Temporal filter ($\mathbf{w}$)');

ah = plt.subplot(4, 2, 3)
plt.plot(time, z, 'k.-')
plt.plot(time[tIdx], z[tIdx], 'bo', ms=8)
plt.plot(time[tIdx+31], z[tIdx+31], 'ro', ms=8)
ah.set_xticklabels([])
ah.set_ylabel('$z$')
ah.set_title('Step 1.')

ah = plt.subplot(4, 2, 5)
plt.plot(time, y, 'k.-')
ah.set_xticklabels([])
ah.set_ylabel('$y$')
ah.set_title('Step 2')

ah = plt.subplot(4, 2, 6)
zVals = np.linspace(-2, 2, 101)
plt.plot(zVals, nonlin(zVals), 'k-')
ah.set_xticks([0])
ah.set_xlabel('$z$')
ah.set_ylabel('$f(z)$')
ah.set_title('Nonlinearity ($f$)')

ah = plt.subplot(4, 2, 7)
plt.plot(time, np.zeros(time.size), 'k-')
for count, tt in zip(spikeCounts, time):
    for spike in np.arange(count):
        r = np.random.rand(1)*dt
        plt.plot([tt+r, tt+r], [-0.1, 0.1], 'k-')
ah.set_yticks([0])
ah.set_xlabel('Time (s)')
ah.set_ylabel('Spikes')
ah.set_title('Step 3')

fig.subplots_adjust(hspace=0.35)
```

### How do you estimate an LN model from data?

The STA will provide the LN model's temporal filter whenever a white noise stimulus is used. For other stimuli, one will either have to fix the nonlinearity beforehand or give up the idea of having a convex optimization problem. In the first case, the nonlinearity is usually fixed to be the identity function or a exponential function, in which case the temporal filter is found by using vanilla linear or Poisson regression, respectively. In the second case, it is possible to find the temporal filter by solving a non-convex optimiation problem which tries to maximize the mutual information between the similarity score and the stimulus. It is thus a bit more work to estimate the LN model for non-white stimuli, and one will have to make a sacrifice by either fixing the nonlinearity or by moving over to solving a non-convex optimization problem.

### Why is the temporal filter sometimes flipped?

Occasionally you may notice that people present temporal filters that are mirrored or flipped, that is, the filter has been flipped along the time dimension as illustrated below.

```python
# plot the stimulus, spikes, and the window containing the stimulus preceeding each spike
plt.figure(figsize=(15, 3))
ah = plt.subplot(1, 2, 1)
plt.plot(t, f, '-', color='gray')
ah.set_ylabel('Filter magnitude')
ah.set_xlabel('Time (s)')
ah.set_title('Original filter');
ah = plt.subplot(1, 2, 2)
plt.plot(-1*t, f, '-', color='gray')
ah.set_xlabel('Time (s)')
ah.set_title('Time mirrored filter');
```

At first glance this might seem a bit odd, as the mirrored filter no longer represents the stimulus that obtains the highest similarity score. However, this way of describing filters derives from the study of linear time-invariant (LTI) systems, where the mirrored filter corresponds to the impulse response. That is, if a cell is linear (or nearly so), then the mirrored filter will describe the response (PSTH) to a brief flash. So, while the original filter describes one point in the stimulus space, the mirrored filter actually describes a cell's response. The original and mirrored filter thus describe very different things and should not be confused with each other, despite their similarity.

Even if you might see filters presented in both ways, my experience is that people tend to use the original way when dealing with LN models. This is also expected, as the presence of the nonlinearity means that the mirrored filter no longer correspond to the impulse response in general (it might still be very close to be impulse response if the system mainly operates in a linear region).

MixedTopics/Temporal filters and receptive fields.ipynb


# Probabilistic Programming Concepts

```python
%matplotlib inline

import os
import glob
from pathlib import Path
import numpy as np
import pandas as pd
import matplotlib as mpl
import matplotlib.pyplot as plt
import seaborn as sns

sns.set_context('notebook', font_scale=1.5)
```

## Bayes theorem and parameter estimation

In  general, the problem is set up like this:

- We have some observed outcomes $y$ that we want to model
- The model is formulated as a probability distribution with some parameters $\theta$ to be estimated
- We want to estimate the posterior distribution of the model parameters given the data
$$
P(\theta \mid y) = \frac{P(y \mid \theta) \, P(\theta)}{\int P(y \mid \theta^*) \, P(\theta^*) \, d\theta^*}
$$
- For formulating a specification using probabilistic programming, it is often useful to think of how we would simulated a draw from the model

## Probabilistic Programming

Statistical objects of interest have the same form as the expectation
$$E[f(\theta)] = {\int f(\theta)p(\theta) d\theta}$$
For example, in addition to the marginal in the denominator of Bayes' theorem, the posterior predictive distribution is
$$p(\hat{y} \mid y) = {\int p(\hat{y} \mid \theta) p(\theta \mid y) d\theta}$$
Probabilistic programming is a way to encode such concepts so they can be automatically calculated

- DSL for model construction, inference and evaluation
- Inference Engines
- PyMC3, PyStan and Edward 2

## Estimating integrals

Integration problems are common in statistics whenever we are dealing with continuous distributions. For example the expectation of a function is an integration problem

$$
E[f(x)] = \int{f(x) \, p(x) \, dx}
$$

In Bayesian statistics, we need to solve the integration problem for the marginal likelihood or evidence

$$
p(X \mid \alpha) = \int{p(X \mid \theta) \, p(\theta \mid \alpha) d\theta}
$$

where $\alpha$ is a hyperparameter and $p(X \mid \alpha)$ appears in the denominator of Bayes theorem

$$
p(\theta | X) = \frac{p(X \mid \theta) \, p(\theta \mid \alpha)}{p(X \mid \alpha)}
$$

In general, there is no closed form solution to these integrals, and we have to approximate them numerically. The first step is to check if there is some **reparameterization** that will simplify the problem. Then, the general approaches to solving integration problems are

1. Numerical quadrature
2. Importance sampling, adaptive importance sampling and variance reduction techniques (Monte Carlo swindles)
3. Markov Chain Monte Carlo
4. Asymptotic approximations (Laplace method and its modern version in variational inference)

This lecture will review the concepts for quadrature and Monte Carlo integration.

## Numerical integration (Quadrature)

You may recall from Calculus that integrals can be numerically evaluated using quadrature methods such as Trapezoid and Simpson's's rules. This is easy to do in Python, but has the drawback of the complexity growing as $O(n^d)$ where $d$ is the dimensionality of the data, and hence infeasible once $d$ grows beyond a modest number.

### Integrating functions

```python
from scipy.integrate import quad
```

```python
def f(x):
    return x * np.cos(71*x) + np.sin(13*x)
```

```python
x = np.linspace(0, 1, 100)
plt.plot(x, f(x))
pass
```

#### Exact solution

```python
from sympy import sin, cos, symbols, integrate

x = symbols('x')
integrate(x * cos(71*x) + sin(13*x), (x, 0,1)).evalf(6)
```

    0.0202549

#### Using quadrature

```python
y, err = quad(f, 0, 1.0)
y
```

    0.02025493910239419

#### Multiple integration

Following the `scipy.integrate` [documentation](http://docs.scipy.org/doc/scipy/reference/tutorial/integrate.html), we integrate

$$
I=\int_{y=0}^{1/2}\int_{x=0}^{1-2y} x y \, dx\, dy
$$

```python
x, y = symbols('x y')
integrate(x*y, (x, 0, 1-2*y), (y, 0, 0.5))
```

    0.0104166666666667

```python
from scipy.integrate import nquad

def f(x, y):
    return x*y

def bounds_y():
    return [0, 0.5]

def bounds_x(y):
    return [0, 1-2*y]

y, err = nquad(f, [bounds_x, bounds_y])
y
```

    0.010416666666666668

## Curse of dimensionality and concentration of measure

```python
plt.plot([0,1,1,0,0], [0,0,1,1,0])
θ = np.linspace(0, 2*np.pi, 100)
plt.scatter(np.random.rand(1000), np.random.rand(1000))
plt.plot(0.5+0.5*np.cos(θ), 0.5+0.5*np.sin(θ))
plt.axis('square')
pass
```

Suppose we inscribe an $d$-dimensional sphere in a $d$-dimensional cube. What happens as $n$ grows large?

The volume of a $d$-dimensional unit sphere is

$$V = \frac{\pi^{\frac{d}{2}}}{\frac{d}{2} \Gamma(\frac{d}{2})}$$

The Gamma function has a factorial growth rate, and hence as $d \to \infty$, $V(d) \to 0$.

In fact, for a sphere of radius $r$, as $d \to infty$, almost all the volume is contained in an annulus of width $r/d$ near the boundary of the sphere. And since the volume of the unit sphere goes to 0 while the volume of unit sphere is constant at 1 while $d$ goes to infinity, essentially all the volume is contained in the corners outside the sphere.

For more explanation of why this matters, see this [doc](https://www.math.ucdavis.edu/~strohmer/courses/180BigData/180lecture1.pdf)

## Working with random variables

See [notebook S14B](S14B_Random_Variables.ipynb)

## Monte Carlo methods

See [notebook S14C](S14C_Monte_Carlo_Methods.ipynb)

## Monte Carlo integration

See [notebook S14D](S14D_Monte_Carlo_Integration.ipynb)

## Drawing pictures

### DBDA diagrams

- [Using LaTeX](https://github.com/tinu-schneider/DBDA_hierach_diagram)
- [Using R and LibreOffice](https://github.com/rasmusab/distribution_diagrams)

Example

### Plate diagrams

- [Using `daft`](http://daft-pgm.org)

```python
import daft
```

#### Coin toss model

```python
pgm = daft.PGM(shape=[2.5, 3.0], origin=[0, -0.5])

pgm.add_node(daft.Node("alpha", r"$\alpha$", 0.5, 2, fixed=True))
pgm.add_node(daft.Node("beta", r"$\beta$", 1.5, 2, fixed=True))
pgm.add_node(daft.Node("p", r"$p$", 1, 1))
pgm.add_node(daft.Node("n", r"$n$", 2, 0, fixed=True))
pgm.add_node(daft.Node("y", r"$y$", 1, 0, observed=True))

pgm.add_edge("alpha", "p")
pgm.add_edge("beta", "p")
pgm.add_edge("n", "y")
pgm.add_edge("p", "y")

pgm.render()
plt.close()
pgm.figure.savefig("bias.png", dpi=300)
pass
```

```python
from IPython.display import Image
```

```python
Image("bias.png", width=400)
```

#### Linear regression model

```python
# Instantiate the PGM.
pgm = daft.PGM(shape=[4.0, 3.0], origin=[-0.3, -0.7])

# Hierarchical parameters.
pgm.add_node(daft.Node("alpha", r"$\alpha$", 0.5, 2))
pgm.add_node(daft.Node("beta", r"$\beta$", 1.5, 2))
pgm.add_node(daft.Node("sigma", r"$\sigma$", 0, 0))

# Deterministic variable.
pgm.add_node(daft.Node("mu", r"$\mu_n$", 1, 1))

# Data.
pgm.add_node(daft.Node("x", r"$x_n$", 2, 1, observed=True))
pgm.add_node(daft.Node("y", r"$y_n$", 1, 0, observed=True))

# Add in the edges.
pgm.add_edge("alpha", "mu")
pgm.add_edge("beta", "mu")
pgm.add_edge("x", "mu")
pgm.add_edge("mu", "y")
pgm.add_edge("sigma", "y")

# And a plate.
pgm.add_plate(daft.Plate([0.5, -0.5, 2, 2], label=r"$n = 1, \cdots, N$",
    shift=-0.1, rect_params={'color': 'white'}))

# Render and save.
pgm.render()
plt.close()
pgm.figure.savefig("lm.png", dpi=300)
```

```python
Image(filename="lm.png", width=400)
```

```python

```


# Realization of Recursive Filters

*This jupyter notebook is part of a [collection of notebooks](../index.ipynb) on various topics of Digital Signal Processing. Please direct questions and suggestions to [Sascha.Spors@uni-rostock.de](mailto:Sascha.Spors@uni-rostock.de).*

## Cascaded Structures

The realization of recursive filters with a high order may be subject to numerical issues. For instance, when the coefficients span a wide amplitude range, their quantization may require a small quantization step or may impose a large relative error for small coefficients. The basic concept of cascaded structures is to decompose a high order filter into a cascade of lower order filters, typically first and second order recursive filters.

### Decomposition into Second-Order Sections

The rational transfer function $H(z)$ of a linear time-invariant (LTI) recursive system can be [expressed by its zeros and poles](introduction.ipynb#Transfer-Function) as

\begin{equation}
H(z) = \frac{b_M}{a_N} \cdot \frac{\prod_{\mu=1}^{P} (z - z_{0\mu})^{m_\mu}}{\prod_{\nu=1}^{Q} (z - z_{\infty\nu})^{n_\nu}}
\end{equation}

where $z_{0\mu}$ and $z_{\infty\nu}$ denote the $\mu$-th zero and $\nu$-th pole of degree $m_\mu$ and $n_\nu$ of $H(z)$, respectively. The total number of zeros and poles is denoted by $P$ and $Q$.

The poles and zeros of a real-valued filter $h[k] \in \mathbb{R}$ are either single real valued or conjugate complex pairs. This motivates to split the transfer function into

* first order filters constructed from a single pole and zero
* second order filters constructed from a pair of conjugated complex poles and zeros

Decomposing the transfer function into these two types by grouping the poles and zeros into single poles/zeros and conjugate complex pairs of poles/zeros results in

\begin{equation}
H(z) = K \cdot \prod_{\eta=1}^{S_1} \frac{(z - z_{0\eta})}{(z - z_{\infty\eta})}
\cdot \prod_{\eta=1}^{S_2} \frac{(z - z_{0\eta}) (z - z_{0\eta}^*)} {(z - z_{\infty\eta})(z - z_{\infty\eta}^*)}
\end{equation}

where $K$ denotes a constant and $S_1 + 2 S_2 = N$ with $N$ denoting the order of the system. The cascade of two systems results in a multiplication of their transfer functions. Above decomposition represents a cascade of first- and second-order recursive systems. The former can be treated as a special case of second-order recursive systems. The decomposition is therefore known as decomposition into second-order sections (SOSs) or [biquad filters](https://en.wikipedia.org/wiki/Digital_biquad_filter). Using a cascade of SOSs the transfer function of the recursive system can be rewritten as

\begin{equation}
H(z) = \prod_{\mu=1}^{S} \frac{b_{0, \mu} + b_{1, \mu} \, z^{-1} + b_{2, \mu} \, z^{-2}}{1 + a_{1, \mu} \, z^{-1} + a_{2, \mu} \, z^{-2}}
\end{equation}

where $S = \lceil \frac{N}{2} \rceil$ denotes the total number of SOSs. These results state that any real valued system of order $N > 2$ can be decomposed into SOSs. This has a number of benefits

* quantization effects can be reduced by sensible grouping of poles/zeros, e.g. such that the spanned amplitude range of the filter coefficients is limited
* A SOS may be extended by a gain factor to further reduce quantization effects by normalization of the coefficients
* efficient and numerically stable SOSs serve as generic building blocks for higher-order recursive filters

### Example - Cascaded second-order section realization of a lowpass

The following example illustrates the decomposition of a higher-order recursive Butterworth lowpass filter into a cascade of second-order sections.

```python
%matplotlib inline
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.markers import MarkerStyle
from matplotlib.patches import Circle
import scipy.signal as sig

N = 9  # order of recursive filter

def zplane(z, p, title='Poles and Zeros'):
    "Plots zero and pole locations in the complex z-plane"
    ax = plt.gca()

    ax.plot(np.real(z), np.imag(z), 'bo', fillstyle='none', ms = 10)
    ax.plot(np.real(p), np.imag(p), 'rx', fillstyle='none', ms = 10)
    unit_circle = Circle((0,0), radius=1, fill=False,
                         color='black', ls='solid', alpha=0.9)
    ax.add_patch(unit_circle)
    ax.axvline(0, color='0.7')
    ax.axhline(0, color='0.7')

    plt.title(title)
    plt.xlabel(r'Re{$z$}')
    plt.ylabel(r'Im{$z$}')
    plt.axis('equal')
    plt.xlim((-2, 2))
    plt.ylim((-2, 2))
    plt.grid()

# design filter
b, a = sig.butter(N, 0.2)
# decomposition into SOS
sos = sig.tf2sos(b, a, pairing='nearest')

# print filter coefficients
print('Coefficients of the recursive part \n')
print(['%1.2f'%ai for ai in a])
print('\n')
print('Coefficients of the recursive part of the individual SOS \n')
print('Section \t a1 \t\t a2')
for n in range(sos.shape[0]):
    print('%d \t\t %1.5f \t %1.5f'%(n, sos[n, 4], sos[n, 5]))

# plot pole and zero locations
plt.figure(figsize=(5,5))
zplane(np.roots(b), np.roots(a), 'Poles and Zeros - Overall')

plt.figure(figsize=(10, 7))
for n in range(sos.shape[0]):
    plt.subplot(231+n)
    zplane(np.roots(sos[n, 0:3]), np.roots(sos[n, 3:6]), title='Poles and Zeros - Section %d'%n)
plt.tight_layout()

# compute and plot frequency response of sections
plt.figure(figsize=(10,5))
for n in range(sos.shape[0]):
    Om, H = sig.freqz(sos[n, 0:3], sos[n, 3:6])
    plt.plot(Om, 20*np.log10(np.abs(H)), label=r'Section %d'%n)

plt.xlabel(r'$\Omega$')
plt.ylabel(r'$|H_n(e^{j \Omega})|$ in dB')
plt.legend()
plt.grid()
```

**Exercise**

* What amplitude range is spanned by the filter coefficients?
* What amplitude range is spanned by the SOS coefficients?
* Change the pole/zero grouping strategy from `pairing='nearest'` to `pairing='keep_odd'`. What changes?
* Increase the order `N` of the filter. What changes?

Solution: Inspecting both the coefficients of the recursive part of the original filter and of the individual SOS reveals that the spanned amplitude range is lower for the latter. The choice of the pole/zero grouping strategy influences the locations of the poles/zeros in the individual SOS, the spanned amplitude range of their coefficients and the transfer functions of the individual sections. The total number of SOS scales with the order of the original filter.

**Copyright**

This notebook is provided as [Open Educational Resource](https://en.wikipedia.org/wiki/Open_educational_resources). Feel free to use the notebook for your own purposes. The text is licensed under [Creative Commons Attribution 4.0](https://creativecommons.org/licenses/by/4.0/), the code of the IPython examples under the [MIT license](https://opensource.org/licenses/MIT). Please attribute the work as follows: *Sascha Spors, Digital Signal Processing - Lecture notes featuring computational examples, 2016-2018*.


#####################################################

# 5. Kernels and RKHS

#####################################################

###########################

## Answer 5 (a)

###########################

```python
import numpy as np
from sklearn.metrics.pairwise import linear_kernel as lk
import matplotlib.pyplot as plt
```

```python
# defined input space
X1 = np.linspace(-1,1,10).reshape(-1,1)
#X1.shape
#X1[0:]
```

```python
def linear_kernel (X,X1):
    lk = np.dot(X,X1)
    return lk
```

```python
'''
k = linear_kernel (1,X1)
print(k)
plt.plot(X1,k)
plt.show()

#############################

def gram_mat (X,X1):
    K= []
    m = X.shape[0]
    for i in range(m):
        K = np.append(K,np.dot (X[i,0],X1))
    K=K.reshape(X.shape[0],X1.shape[0])
    return K
K = gram_mat (X1,X1)
plt.matshow(K)
K[0,:]

'''
```

    '\nk = linear_kernel (1,X1)\nprint(k)\nplt.plot(X1,k)\nplt.show()\n\n#############################\n\ndef gram_mat (X,X1):\n    K= []\n    m = X.shape[0]\n    for i in range(m):\n        K = np.append(K,np.dot (X[i,0],X1))\n    K=K.reshape(X.shape[0],X1.shape[0])\n    return K  \nK = gram_mat (X1,X1)\nplt.matshow(K)\nK[0,:]\n\n'

```python
k =  linear_kernel (1,X1)
k_func1 = 3 * k
k_func2 = 3 * k + 4
k_func3 = 3 * k + 4 * k
```

```python
plt.plot(X1,k_func1,label='generated func : $ g(k) = 3k $')
plt.plot(X1,k_func2,label='generated func : $ g(k) = 3k+4 $')
plt.plot(X1,k_func3,label='generated func : $ g(k) = 3k+4k $')
plt.plot(X1,k,'.',label = 'linear kernel function')
plt.legend()
plt.show()
```

$||f||_{k}$

$||f||^2_k = <f,f>_k = <k_w,k_w>_k = k(w,w)$

function 1:
   $ k_1(w,w) = 3 k(w,w) = 3 w^2  < \infty $

function 2:
   $ k_2(w,w) = 3 k(w,w)+4 = 3 w^2 +4 < \infty $

function 3:
   $ k_3(w,w) = 3 k(w,w)+4 k(w,w) = 3 w^2 + 4 w^2 < \infty$

##################################

## Answer 5 (b)

###################################

```python
def gaussian_kernel (X,X1,p):
    gk = np.exp (-1 *( (X-X1)**2 ) / (2*(p**2) ) )
    return gk
```

```python
'''
for i in range(X1.shape[0]):
    k = gaussian_kernel (X1[i,0],X1,p[0,1])
    plt.plot(k)
'''
```

    '\nfor i in range(X1.shape[0]):\n    k = gaussian_kernel (X1[i,0],X1,p[0,1])\n    plt.plot(k)\n'

### For each bandwidth $\rho \in \{ 0.1, 0.5, 1 \} $ function hase been generated

```python
# given
p = np.matrix([0.1,0.5,1])

# defined input space for given limit X = [-1,1]
X1 = np.linspace(-1,1,100).reshape(-1,1)
X2 = X1

for i in range(p.shape[1]):
    k = gaussian_kernel (X1[-1,0],X2,p[0,i])
    plt.plot(X1,k , label = 'p = %s'%p[0,i])
plt.xlabel('$\mathbb{X2}$')
plt.title('kernel function k(X1,X2) plotted as a functino of X2 dor X1=1')
plt.legend()
plt.show()
```

```python

```

### We're approximating the feature mapping  $\phi(.)$ using taylor series expansion up to d=150 features

$\gamma = \frac{1}{2 \sigma^2}$
\begin{align}
    e^{- \gamma ||x_i - x_j ||} = & e^{- \gamma(x_i-x_j)^2} = e^{-\gamma x_i^2 + 2\gamma x_i x_j-\gamma x_j^2}  \\
                                = & e^{-\gamma x_i^2 \gamma x_j^2} \Bigg(1 \times 1 + \sqrt{\frac{2\gamma}{1!}}x_i \times \sqrt{\frac{2\gamma}{1!}}x_j + \sqrt{\frac{(2\gamma)^2}{2!}}x_i^2 \times \sqrt{\frac{(2\gamma)^2}{2!}}x_j^2 \nonumber \\ & + \sqrt{\frac{(2\gamma)^3}{3!}}x_i^3 \times \sqrt{\frac{(2\gamma)^3}{3!}}x_j^3 + .......  \Bigg) \\
                                = & \phi(x_i)^T\phi(x_j)
\end{align}
here $\phi(x) = e^{-\gamma x^2} \Bigg[ 1, \sqrt{\frac{(2\gamma)^1}{1!}}x^1 , \sqrt{\frac{(2\gamma)^2}{2!}}x^2 ..... \sqrt{\frac{(2\gamma)^{150} }{150!}}x^{150} \Bigg]^T$

```python

```

### ploting gaussian basis function $\phi(x_i)$, where $x_i$  = 1

```python
def gaussian_basis(x,p,d):
    gamma = 1/(2*(p**2))
    g = np.matrix(np.zeros([d+1,1]))
    for i in range(d+1):
        from scipy.special import factorial
        g[i,0] = np.sqrt(((2*gamma)**i)/factorial(i, exact=True))*(x**i)
    g_basis = np.exp(-gamma*(x)**2)*g
    return g_basis

d = 150
g_basis=gaussian_basis(X1[-1,0],p[0,1],d) #gaussian basis for Xi = 1
plt.plot(g_basis)
```

### Plotting function for fixed $\rho$=0.5

```python
d = 150
x1=X1[-1,0]
g_basis1=gaussian_basis(x1,p[0,1],d) # for fixed p=0.5, X1=1

func = np.matrix(np.zeros([X1.shape[0],1]))
for k in range(X1.shape[0]):
    x2 = X2[k,0]
    g_basis2=gaussian_basis(x2,p[0,1],d) # for fixed p
    func[k,:]=np.dot(g_basis1.T,g_basis2)
plt.plot(X1,func,label='p = %s'%p[0,1])
plt.xlabel('$\mathbb{X2}$')
plt.ylabel('k(x1,x2)')
plt.title('x1 = 1')
plt.legend()
plt.show()
```

### ploting functions for $\rho \in \{ 0.1, 0.5, 1 \} $

```python
x1=X1[-1,0]
d = 120

# evaluate kernel funciton for different bandwidth
for m in range (p.shape[1]):
    g_basis1=gaussian_basis(x1,p[0,m],d)
    func = np.matrix(np.zeros([X1.shape[0],1]))

    # get kernel function as a function of X2 (when bandwidth P and X1 fixed)
    for k in range(X1.shape[0]):
        x2 = X2[k,0]
        g_basis2=gaussian_basis(x2,p[0,m],d)
        func[k,:]=np.dot(g_basis1.T,g_basis2)
    plt.plot(X1,func,label='p = %s'%p[0,m])
    plt.xlabel('$\mathbb{X2}$')
    plt.ylabel('k(x1,x2)')
    plt.title('kernel function k(X1,X2) plotted as a functino of X2 dor X1=1')
    plt.legend()
plt.show()
```

```python

```

## Answer 5(c)

```python
import pandas as pd
from sklearn.gaussian_process import GaussianProcessRegressor
```

```python
df1 = pd.read_csv('hw1-q5x.csv',delim_whitespace =True,header=None,dtype=np.float)
df2 = pd.read_csv('hw1-q5y.csv',delim_whitespace =True,header=None,dtype=np.float)
Data=pd.concat([df1,df2],axis=1)
Data.columns=['X','Y']
no_data = df1.shape[0]# Noise free training data
X_train = np.array([-4, -3, -2, -1, 1]).reshape(-1, 1)
Y_train = np.sin(X_train)
```

```python
X_train = df1.values
y_train = df2.values.reshape(-1)
```

```python
X_train.reshape(-1).shape
```

    (10,)

```python
# Instanciate a Gaussian Process model
GaussianProcessRegressor.fit(X_train,y_train)
```

```python
sample_y(X, n_samples=1, random_state=0
```

```python

```

```python

from sklearn.datasets import make_friedman2
from sklearn.gaussian_process import GaussianProcessRegressor
from sklearn.gaussian_process.kernels import DotProduct, WhiteKernel
X, y = make_friedman2(n_samples=500, noise=0, random_state=0)
```

```python
y.shape
```

    (500,)

```python
w = np.linalg.inv((phi.T*phi)+lmbda*I) * (phi.T * y)
```

```python
x = np.array( [ [1,2,3],[2,4,4] ] )
```

```python
x
```

    array([[1, 2, 3],
           [2, 4, 4]])

```python
from sklearn.datasets import make_friedman2
from sklearn.gaussian_process import GaussianProcessRegressor
from sklearn.gaussian_process.kernels import RBF

gpr = GaussianProcessRegressor.fit(X_train, y_train)

```

```python
kernel
```

    1**2 * RBF(length_scale=10)

```python
X_train.shape
```

    (10, 1)

```python
from __future__ import division
import numpy as np
import matplotlib.pyplot as pl

""" This is code for simple GP regression. It assumes a zero mean GP Prior """

# This is the true unknown function we are trying to approximate
f = lambda x: np.sin(0.9*x).flatten()
#f = lambda x: (0.25*(x**2)).flatten()

# Define the kernel
def kernel(a, b):
    """ GP squared exponential kernel """
    kernelParameter = 0.1
    sqdist = np.sum(a**2,1).reshape(-1,1) + np.sum(b**2,1) - 2*np.dot(a, b.T)
    return np.exp(-.5 * (1/kernelParameter) * sqdist)

N = 10         # number of training points.
n = 50         # number of test points.
s = 0.00005    # noise variance.

# Sample some input points and noisy versions of the function evaluated at
# these points.
X = X_train
y = y_train

K = kernel(X, X)
L = np.linalg.cholesky(K + s*np.eye(N))

# points we're going to make predictions at.
Xtest = np.linspace(-5, 5, n).reshape(-1,1)

# compute the mean at our test points.
Lk = np.linalg.solve(L, kernel(X, Xtest))
mu = np.dot(Lk.T, np.linalg.solve(L, y))

# compute the variance at our test points.
K_ = kernel(Xtest, Xtest)
s2 = np.diag(K_) - np.sum(Lk**2, axis=0)
s = np.sqrt(s2)

# PLOTS:
pl.figure(1)
pl.clf()
pl.plot(X, y, 'r+', ms=20)
pl.plot(Xtest, f(Xtest), 'b-')
pl.gca().fill_between(Xtest.flat, mu-3*s, mu+3*s, color="#dddddd")
pl.plot(Xtest, mu, 'r--', lw=2)
pl.savefig('predictive.png', bbox_inches='tight')
pl.title('Mean predictions plus 3 st.deviations')
pl.axis([-5, 5, -3, 3])

# draw samples from the prior at our test points.
L = np.linalg.cholesky(K_ + 1e-6*np.eye(n))
f_prior = np.dot(L, np.random.normal(size=(n,10)))
pl.figure(2)
pl.clf()
pl.plot(Xtest, f_prior)
pl.title('Ten samples from the GP prior')
pl.axis([-5, 5, -3, 3])
pl.savefig('prior.png', bbox_inches='tight')

# draw samples from the posterior at our test points.
L = np.linalg.cholesky(K_ + 1e-6*np.eye(n) - np.dot(Lk.T, Lk))
f_post = mu.reshape(-1,1) + np.dot(L, np.random.normal(size=(n,10)))
pl.figure(3)
pl.clf()
pl.plot(Xtest, f_post)
pl.title('Ten samples from the GP posterior')
pl.axis([-5, 5, -3, 3])
pl.savefig('post.png', bbox_inches='tight')

pl.show()
```

```python
def kernel_func( X1, X2, l=1.0, sigma_f=1.0 ,lmbda ):
    ''' Isotropic squared exponential kernel. Computes a covariance matrix from points in X1 and X2. Args: X1: Array of m points (m x d). X2: Array of n points (n x d). Returns: Covariance matrix (m x n). '''
    sqdist = np.sum(X1**2, 1).reshape(-1, 1) + np.sum(X2**2, 1) - 2 * np.dot(X1, X2.T)
    return (sigma_f**2/lmbda) * np.exp(-0.5 / l**2 * sqdist)
```

```python
 K = kernel(X_train, X_train, l, sigma_f ,lmbda) + sigma_y**2 * np.eye(len(X_train))
```

```python
K.shape
```

    (10, 10)

```python
import numpy as np
l = 1.0
sigma_f = 1.0
def kernel(X1, X2, l, sigma_f ,lmbda ):
    sqdist = np.sum(X1**2, 1).reshape(-1, 1) + np.sum(X2**2, 1) - 2 * np.dot(X1, X2.T)
    return (sigma_f**2/lmbda) * np.exp(-0.5 / l**2 * sqdist)
```

```python
from numpy.linalg import inv
l=1.0
sigma_f=1.0
sigma_y=1e-8
def posterior_predictive(X_s, X_train, Y_train, l, sigma_f, sigma_y, lmbda):
    ''' Computes the suffifient statistics of the GP posterior predictive distribution from m training data X_train and Y_train and n new inputs X_s. Args: X_s: New input locations (n x d). X_train: Training locations (m x d). Y_train: Training targets (m x 1). l: Kernel length parameter. sigma_f: Kernel vertical variation parameter. sigma_y: Noise parameter. Returns: Posterior mean vector (n x d) and covariance matrix (n x n). '''
    K = kernel(X_train, X_train, l, sigma_f ,lmbda) + sigma_y**2 * np.eye(len(X_train))
    K_s = kernel(X_train, X_s, l, sigma_f,lmbda)
    K_ss = kernel(X_s, X_s, l, sigma_f,lmbda) + 1e-8 * np.eye(len(X_s))
    K_inv = inv(K)

    # Equation (4)
    mu_s = K_s.T.dot(K_inv).dot(Y_train)

    # Equation (5)
    cov_s = K_ss - K_s.T.dot(K_inv).dot(K_s)

    return mu_s, cov_s
```

```python

X = np.linspace(-1,1,10).reshape(-1,1)
```

```python

lmbda = 1
# Compute mean and covariance of the posterior predictive distribution
mu_s, cov_s = posterior_predictive(X, X_train, Y_train,l, sigma_f, sigma_y, lmbda)

samples = np.random.multivariate_normal(mu_s.ravel(), cov_s, 3)
plot_gp(mu_s, cov_s, X, X_train=X_train, Y_train=Y_train, samples=samples)
```

```python

```


```python
import numpy as np
import scipy as sp
import scipy.ndimage
import copy
import seaborn as sns
import time
import matplotlib.pyplot as plt
from mpmath import mpf, mpc, mp, log
```

# S&B 4.9 - Gambler's Problem

For reference the update rule for value iteration is:

$$ V_{k+1}(s) = \max_{a} \sum_{s', r} p(s', r| s, a) \left[ r + \gamma \cdot V_k(s') \right]$$

We ran the simulation and found that:

+ For $p_h = 0.45$ we replicate the results in the book. If we wanted to replicate the policy graph on the curve we would need to account for numerical error and simply take the **minimum** argmax of the Q function. The argmax should in theory return many different possibilities, and the minimum of these would produce the policy in the book. However, as mentioned in the book there are multiple "optimality" contours per se in the value function.

    + Some of the contours focus on gambling the minimal amount that will get us to the next quartile (similar to what we see in the book).

    + Other contours focus on gambling **everything**. This strategy makes sense because it minimizes the number of coin flips we make.

    + From the graphs of $Q(s,a)$ and $Q(a|s)$ we can see that these optimal strategies form a set of diamonds, with up and to the right edges of the diamonds representing "betting it all trajectories" and up and to the left edges representing "betting just enough to get to the next quartile" strategies.

+ For $p_h = 0.5$ the value function is basically $V(s) = s/100$ and when $p_h > 0.5$ as $p_h$ increases the value function gets closer and closer to a square (meaning the likelihood of winning gets closer and closer to 1 for low $s$ states).

+ As $p_h$ gets smaller and smaller the kinks in the value function get more and more extreme and defined. In the limit of extremely small $p_h$ it becomes essentially a step function. The step jumps occur when we have enough "coins" so that we reduce the integral number of flips needed to win. The heights of the steps correlate with $p_h^n$ where $n$ is the number of consecutive flips needed to win.

We'll leave our analysis at that.

```python
mp.dps = 200

win_threshold = 100
p_heads = mpf(1)*4/10
log_theta = -200
V = np.array([mpf(0) for i in range(win_threshold + 1)])
pi = np.zeros(win_threshold + 1)
Q = np.array([[mpf(0) for i in range(win_threshold + 1)] for j in range(win_threshold + 1)])

def get_states_and_rewards(s):
    actions = np.arange(1, s + 1, dtype=int)
    win_cond = (s + actions) >= win_threshold
    lose_cond = (s - actions) <= 0
    reward = np.array([mpf(int(cond)) for cond in win_cond])
    winning_states = np.where(win_cond, win_threshold, s + actions)
    losing_states = np.where(lose_cond, 0, s - actions)

    return actions, reward, winning_states, losing_states

counter = 0
t_i = time.time()

while True:
    log_delta = -mp.inf
    for s in range(1, win_threshold):
        actions, reward, winning_states, losing_states = get_states_and_rewards(s)
        v = V[s]
        V[s] = np.max(p_heads * (reward + V[winning_states]) + (1 - p_heads) * (V[losing_states]))

        log_delta = max(log_delta, log(abs(V[s] - v)))

    counter += 1

    if counter % 10 == 0:
        t_f = time.time()
        delta_t = round(t_f - t_i, 2)
        print("Iteration #{} complete in {} s. Current Log Delta: {} ".format(counter, delta_t, log_delta))
        t_i = t_f
    if log_delta < log_theta:
        for s in range(1, win_threshold):
            actions, reward, winning_states, losing_states = get_states_and_rewards(s)
            Q_sa = p_heads * (reward + V[winning_states]) + (1 - p_heads) * (V[losing_states])

            for action, q_value in zip(actions, Q_sa):
                Q[s, action] = q_value

            pi[s] = np.argmax(Q_sa)

        print("Optimization completed after {} iterations".format(counter))
        break
```

```python
fig = plt.figure(figsize=(20,10))

ax0 = fig.add_subplot(121)
plt.plot(V[1:100])
plt.xlabel("Current Stash Amount", size=20)
plt.xticks(size=15)
plt.ylabel("State Value", size=20)
plt.yticks(size=15)
ax1 = fig.add_subplot(122)
plt.bar(x=np.arange(1,100),height=pi[1:100])
plt.xlabel("Current Stash Amount", size=20)
plt.xticks(size=15)
plt.ylabel("Wager Amount", size=20)
plt.yticks(size=15)
plt.tight_layout()
```

```python
fig, ax = plt.subplots(figsize=(15,10))
mp_to_float = np.vectorize(float)
sns.heatmap(mp_to_float(Q.T))
plt.title("Action Value Function - Q(s,a)", size=30)
plt.xlabel("s", size=20)
plt.ylabel("a", size=20)
plt.tight_layout()
plt.show()
```

```python
fig = plt.figure(figsize=(20,20))

q_max = np.max(Q[s][0:25])
max_list = []

for s in np.arange(0, 25):
    plt.plot(Q[s][0:25], linewidth=2)
    q_smax = np.max((mp_to_float(Q[s])))
    plt.hlines(q_smax, 0, 25, linestyles='dashed')
    max_list.append(q_smax)
    plt.vlines(s, 0, q_max, linestyles='dashed')

plt.plot(max_list, color='black', linewidth=5)
plt.plot(np.arange(25, 0, -1), max_list, color='black', linewidth=5)

plt.legend(np.arange(0, 25))
plt.tight_layout()
```

```python
fig = plt.figure(figsize=(20,20))

q_max = np.max(Q[s][1:51])
max_list = []

for s in np.arange(0, 51):
    plt.plot(range(0,51), Q[s][0:51], linewidth=2)
    q_smax = np.max((mp_to_float(Q[s])))
    plt.hlines(q_smax, 0, 51, linestyles='dashed')
    max_list.append(q_smax)
    plt.vlines(s, 0, q_max, linestyles='dashed')

#plt.plot(max_list, color='black', linewidth=5)
#plt.plot(np.arange(50, 25, -1), max_list, color='black', linewidth=5)

plt.legend(np.arange(0, 51))
plt.tight_layout()
```

# S&B 5.12 - Racetrack

## I. Track Generation

The first thing is that we need to create a racetrack. The simplest representation for the racetrack is simply an $N \times N$ matrix in which an entry is 1 if it's outside the track and zero otherwise. We generate tracks as follows:

+ We initialize the track to an $N \times N$ grid where every square in the grid is on the track.
+ We take out rectangles from the upper left and lower right of the track. The sizes of the rectangles are randomly sampled, with the length of each dimension $L$ being distributed as $L \sim RandInt[Floor(N/4), \ Floor(N/2)]$. Taking out these rectangles creates the "right" turn at the end requested by the authors, it also creates left turn for fun.
+ We then iterate through each cell on the grid, check if it's outside the grid, and if it is we flip it to being on the track with probability equal to the percentage of its neighbors on the track. This (approximately) guarantees that only cells at the edge of the track get moved into the track.
+ This iteration is repeated some number of times so we get some interesting shapes due to the randomness of the sampling. We do smoothing after the final iteration to attempt to eliminate blocks that are technically "on the track" but are inaccessible due to being surrounded by blocks "off the track".

Here is the code -- if you run it you can see we get some nice tracks with semi-smooth curves.

```python
N = 25
repeats = 4
filter_threshold = 0.2
filter_size = 3

base_track = np.zeros((N, N), dtype=float)

upper_left_rect = np.random.randint(int(N/4), int(3*N/4)), np.random.randint(N - int(3*N/4), N - int(N/4))
lower_right_rect = np.random.randint(int(N/4), int(3*N/4)), np.random.randint(int(N/4), int(3*N/4))

for i in range(N):
    for j in range(N):
        if i < upper_left_rect[0] and j > upper_left_rect[1]:
            base_track[i, j] = 1

        if i > N - lower_right_rect[0] and j < lower_right_rect[1]:
            base_track[i, j] = 1

for repeat in range(repeats):
    for i in range(N):
        for j in range(N):
            if base_track[i, j] == 1:
                neighs = np.array([base_track[i + n, j + m] for n in np.arange(-1, 2) for m in np.arange(-1, 2)
                                   if 0 <= i + n < N and 0 <= j + m < N])

                neighs_sum = np.sum(neighs)
                is_flipped = np.random.binomial(1, 1 - np.mean(neighs))

                if is_flipped:
                    base_track[i, j] = 0

track = (scipy.ndimage.filters.uniform_filter(base_track, filter_size) > filter_threshold)

fig, ax = plt.subplots(figsize=(15,10))

sns.heatmap(track.T, ax=ax)
plt.xlabel("X coordinate", size=20)
plt.ylabel("Y coordinate", size=20)
plt.tight_layout()
plt.show()
```

Note that the indices of the track array correspond to the x and y coordinates respectively. We only transpose when plotting so that the plot aligns with our intuition.

## II. Trajectory Sampling

First let's formulate the problem concretely. The state space is the set of possible x and y coordinates and x and y velocities. Let $X_N$ be the set of all coordinates in the grid, $X_{off}$ be the set of all off-track coordinates, $X_S$ the set of start coordinates, and $V$ be the set of all possible velocities. Then $X_N = [0, N] \times [0, N]$ where these are integer intervals, and the set of possible spatial states is the quotient $X_N - X_{off}$ since all off-track coordinates transition you to a start state. Also we have that $V = [0, v_{x, max}] \times [0, v_{y, max}]$ where $v_{i, max} = 5$ in our instance of the problem. Finally the actual set of possible states is:

$$ \mathscr{S} = \left( X_S \times \{ \vec{0} \} \right) \ \bigcup \ \left( X_N - X_{off} - X_S \right) \times \left( V -  \{ \vec{0} \} \right) $$

Let us denote a particular state in terms of the spatial and velocity coordinates, $\left( \vec{x}, \vec{v} \right) \in \mathscr{S}$. Then if $\vec{v} = (v_x, v_y)$ and $H$ is the Heaviside step function with the convention that $H(0) = 1$:

$$ \mathscr{A}(\vec{x}, \vec{v}) = \left\{ -H(v_y - 1), 0, 1 -  H(v_x - v_{x,max}) \right\} \times \left\{-H(v_y - 1),0, 1 - H(v_y - v_{y,max}) \right\},  \forall \left( \vec{x}, \vec{v} \right) \in \mathscr{S} $$

Note that here we rely on the convention that duplicate values in a set are not distinct. Finally, if $\vec{a} \in \mathscr{A}(\vec{x}, \vec{v})$ is the action vector then the environment acts as:

$$
\begin{equation}
  (\vec{X}', \vec{V}') =\begin{cases}
    (\vec{X} + \vec{V} + \vec{A}, \vec{V} + \vec{A}), & \text{with probability } 1 - \delta\\
    (\vec{X} + \vec{V}, \vec{V}), & \text{with probability } \delta
  \end{cases}
\end{equation}
$$

In our particular case $\delta=0.1$. Note that we are choosing to have the action $\vec{A}$ apply immediately on the current state, so that it determines the next state if successful. Note that this equation only applies exactly if the newly calculated state is on the track -- otherwise the new state is the starting state.

The task is to determine the optimal policy from each starting state. This means we start each episode always from the starting states and simulate our trajectories in this manner.

```python
vx_max = 5
vy_max = 5
v_max = [vx_max, vy_max]
```

```python
def get_start_states(track):
    N = len(track)
    x_start = []
    for x in range(N):
        if track[x, 0] == 0:
            x_start.append(np.array([x, 0]))

    return np.array(x_start)

def get_end_states(track):
    N = len(track)
    y_end = []
    for y in range(N):
        if track[N - 1, y] == 0:
            y_end.append(np.array([N - 1, y]))

    return np.array(y_end)

def valid_actions(v_x, v_y, v_max):
    all_actions = [(i, j) for i in range(-1, 2) for j in range(-1, 2)]
    actions = [(i, j) for i, j in all_actions
               if (0 <= v_x + i <= v_max[0]) and (0 <= v_y + j <= v_max[1])
                   and  (v_x + i, v_y + j) != (0, 0)]
    return actions

class PolicyClass():

    def __init__(self, track, epsilon, v_max=v_max):
        """
        We represent the policy as a dictionary indexed by
        (valid) coordinate tuples. Each value of the dictionary
        is a dictionary whose keys correspond to an allowed velociy
        state.

        Thus self.policy[(x, y)][(v_x, v_y)] returns the policy [a_x, a_y]
        for some state with a caveat.
        """
        self.start_states = get_start_states(track).tolist()
        self.epsilon = epsilon
        N = len(track)
        self.policy = {}
        self.v_max = v_max

        for i in range(N):
            for j in range(N):
                if track[i, j] == 0 and [i, j] not in self.start_states:
                    self.policy[(i, j)] = {}

                    for v_x in range(0, v_max[0] + 1):
                        for v_y in range(0, v_max[1] + 1):
                            if v_x == 0 and v_y == 0:
                                continue

                            if v_x == v_max[0] and v_y != v_max[1]:
                                self.policy[(i, j)][(v_x, v_y)] = np.array([0 , 1])
                            elif v_x != v_max[0] and v_y == v_max[1]:
                                self.policy[(i, j)][(v_x, v_y)] = np.array([1 , 0])
                            elif v_x == v_max[0] and v_y == v_max[1]:
                                self.policy[(i, j)][(v_x, v_y)] = np.array([0 , 0])
                            else:
                                self.policy[(i, j)][(v_x, v_y)] = np.array([1 , 1])

                elif track[i, j] == 0 and [i, j] in self.start_states:
                    self.policy[(i, j)] = {(0,0): np.array([1, 1])}
                else:
                    continue

    def update_action(self, new_a_x, new_a_y, x, y, v_x, v_y):
        if new_a_x not in [-1, 0, 1] or new_a_y not in [-1, 0, 1]:
            raise ValueError("a_x and a_y must be one of `[-1, 0, 1]`")

        if v_x == v_max[0] and new_a_x == 1:
            raise ValueError("a_x cannot be 1 when v_x is already at its maximum value")

        if v_y == v_max[1] and new_a_y == 1:
            raise ValueError("a_y cannot be 1 when v_y is already at its maximum value")

        if v_x == 0 and new_a_x == -1:
            raise ValueError("a_x cannot be -1 when v_x is already 0")

        if v_y == 0 and new_a_y == -1:
            raise ValueError("a_y cannot be -1 when v_y is already 0")

        if [x,y] in self.start_states and [v_x, v_y] != [0, 0]:
            raise ValueError("The velocity at a start location must be zero")

        self.policy[(x, y)][(v_x, v_y)] = np.array([new_a_x, new_a_y], dtype=int)

    def get_action(self, x, y, v_x, v_y):
        return self.policy[(x, y)][(v_x, v_y)]

    def sample(self, x, y, v_x, v_y):
        """
        Note the policy is actually epsilon greedy, but we
        keep track of the deterministic part of it
        """
        if [x, y] not in self.start_states:
            actions = valid_actions(v_x, v_y, v_max)
        else:
            # Any starting velocity MUST leave the start line, just makes things simpler
            actions = [(i,j) for i in [0, 1] for j in [1]]

        num_actions = len(actions)

        is_greedy = np.random.binomial(1, 1 - self.epsilon + self.epsilon/num_actions)

        if is_greedy:
            return self.get_action(x, y, v_x, v_y)
        else:
            action_index = np.random.randint(num_actions)

            a_x = actions[action_index][0]
            a_y = actions[action_index][1]
            return np.array([a_x, a_y])

class QClass():

    def __init__(self, track, v_max=v_max, init=0.0):
        """
        We represent the action-value function as a dictionary indexed by
        (valid) coordinate tuples. Each value of the dictionary
        is a dictionary with keys corresponding to a combination
        of allowed velociy states. The values of the velocity dictionary
        are in turn dictionaries whose keys are valid action tuples

        Thus self.Q[(x, y)][(v_x, v_y)][(a_x, a_y)] returns the value
        Q(s=(x,y,v_x,v_y), a=(a_x, a_y)) for some state and action combination.

        We also keep track of counts in a similar tensor so we can calcualte the
        MC average incrementally.
        """

        self.start_states = get_start_states(track).tolist()
        N = len(track)
        self.Q = {}
        self.counts = {}
        self.v_max = v_max

        for i in range(N):
            for j in range(N):
                if track[i, j] == 0 and [i, j] not in self.start_states:
                    self.Q[(i, j)] = {}
                    self.counts[(i, j)] = {}

                    for v_x in range(0, v_max[0] + 1):
                        for v_y in range(0, v_max[1] + 1):

                            if v_x == 0 and v_y == 0:
                                continue

                            actions = valid_actions(v_x, v_y, v_max)

                            self.Q[(i, j)][(v_x, v_y)] = {a: init for a in actions}
                            self.counts[(i, j)][(v_x, v_y)] = {a: 0 for a in actions}

                elif track[i, j] == 0 and [i, j] in self.start_states:
                    # Any starting velocity MUST leave the start line, just makes things simpler
                    start_actions = [(0, 1), (1, 1)]
                    self.Q[(i, j)] = {(0,0): {a: init for a in start_actions}}
                    self.counts[(i, j)] = {(0,0): {a: 0 for a in start_actions}}
                else:
                    continue

    def update(self, G, x, y, v_x, v_y, a_x, a_y):
        n = self.get_n(x, y, v_x, v_y, a_x, a_y)
        q = self.get_q(x, y, v_x, v_y, a_x, a_y)

        self.Q[(x, y)][(v_x, v_y)][(a_x, a_y)] = q + (1/(n + 1))*(G - q)
        self.counts[(x, y)][(v_x, v_y)][(a_x, a_y)] = n + 1

    def get_q_s(self, x, y, v_x, v_y):
        return self.Q[(x, y)][(v_x, v_y)]

    def get_q(self, x, y, v_x, v_y, a_x, a_y):
        return self.get_q_s(x, y, v_x, v_y)[(a_x, a_y)]

    def get_n(self, x, y, v_x, v_y, a_x, a_y):
        return self.counts[(x, y)][(v_x, v_y)][(a_x, a_y)]

    def get_max_action(self, x, y, v_x, v_y):
        q_s = self.get_q_s(x, y, v_x, v_y)
        max_a = None
        max_q_s = -np.inf

        for a, q in q_s.items():
            if q > max_q_s:
                max_q_s = q
                max_a = a

        return np.array(max_a)

def generate_trajectory(pi, track, delta):
    """
    Samples from pi to generate a simulated trajectory
    Handles the logic for determining state transitions
    including going off track, enviroment noise and episode
    termination.

    The finish state is defined as crossing the finishing line
    WITHOUT going above or below it.
    """
    start_index = np.random.randint(len(pi.start_states))
    X, V = np.array(copy.deepcopy(pi.start_states[start_index])), np.array([0, 0])
    end_states = get_end_states(track)
    end_x = end_states[0][0]
    end_y_range = (end_states[0][1], end_states[len(end_states) - 1][1])

    trajectory = []

    while True:
        try:
            A = pi.sample(X[0], X[1], V[0], V[1])
        except:
            print("SAMPLING ERROR")
            print(trajectory)
            print(X, V)
            raise

        trajectory.append((copy.deepcopy(X), copy.deepcopy(V), copy.deepcopy(A)))

        is_noise = np.random.binomial(1, delta)
        if is_noise:
            pass
        else:
            V += A

        X += V

        try:
            is_on_track = not track[X[0], X[1]]
            finish_state = False
        except IndexError:
            is_on_track = False
            finish_state = X[0] > end_x and (end_y_range[0] <= X[1] <= end_y_range[1])

        if not is_on_track and not finish_state:
            start_index = np.random.randint(len(pi.start_states))
            X, V = np.array(copy.deepcopy(pi.start_states[start_index])), np.array([0, 0])
        elif not is_on_track and finish_state:
            return trajectory
        elif is_on_track and not finish_state:
            continue
        elif is_on_track and finish_state:
            raise ValueError("Agent should never be on track and in a finish state")

def update_q_and_pi(trajectory, Q, pi):
    """
    Recursively calculates G based on the trajectory
    Uses G to update Q, and then Q to update pi.

    Note that our task is undiscounted so we don't
    reference gamma. Also the reward is always -1
    for every step
    """
    trajectory.reverse()
    G = 0

    for X, V, A in trajectory:
        G = G - 1
        try:
            Q.update(G, X[0], X[1], V[0], V[1], A[0], A[1])
        except KeyError:
            print("Q UPDATE ERROR")
            print(trajectory)
            print(X, V, A)
            raise
        new_A = Q.get_max_action(X[0], X[1], V[0], V[1])
        pi.update_action(new_A[0], new_A[1], X[0], X[1], V[0], V[1])

def on_policy_mc_control(track, N, epsilon, delta, v_max=v_max, init=0.0, update_iter=1000):
    """Initialization is handled internally by each class"""
    pi = PolicyClass(track, epsilon, v_max=v_max)
    Q = QClass(track, v_max=v_max, init=0.0)

    t_i = time.time()
    total_time = 0

    for iteration in range(N):
        if (iteration + 1) % update_iter == 0:
            t_f = time.time()
            print("Iterations {} through {} complete: {} secs".format(iteration + 1 - update_iter,
                                                                      iteration + 1,
                                                                      round((t_f - t_i), 2)))
            total_time += (t_f - t_i)
            t_i = time.time()

        trajectory = generate_trajectory(pi, track, delta)
        update_q_and_pi(trajectory, Q, pi)

    print("Total Running Time: {} mins".format(round(total_time/60, 2)))

    return Q, pi
```

```python
Q, pi = on_policy_mc_control(track, 10**6, 0.1, 0.1, update_iter=5 * 10**4)
```

```python
# FIXME -- Need better collision detection. Currently just checking
# if car lands on (not crosses) an off-track area. Also may be worth
# trying training with less simulations and having a termination
# condition on the simulator just in case.
trajectory = generate_trajectory(pi, track, 0)

spatial_trajectory = [state[0].tolist() for state in trajectory]
velocities = [state[1].tolist() for state in trajectory]
accel = [state[2].tolist() for state in trajectory]

traj_steps = len(spatial_trajectory)
track_and_path = np.array(copy.deepcopy(track), dtype=float)
counter = 0

for i in range(len(track)):
    for j in range(len(track)):
        if [i, j] in spatial_trajectory:
            index = spatial_trajectory.index([i, j])
            track_and_path[i, j] = 0.25 + (0.75-0.25)/traj_steps * counter
            counter += 1

fig, ax = plt.subplots(figsize=(15,10))
sns.heatmap(track_and_path.T, ax=ax)

for i in range(len(track)):
    for j in range(len(track)):
        if [i, j] in spatial_trajectory:
            index = spatial_trajectory.index([i, j])
            try:
                plt.arrow(i , j,
                          velocities[index + 1][0],
                          velocities[index + 1][1],
                          width=0.1, head_width=0.5,
                          color="blue",alpha=0.9)
                plt.arrow(i , j,
                          accel[index][0],
                          accel[index][1],
                          width=0.02, head_width=0.2,
                          color="mediumseagreen",alpha=0.9)
            except IndexError:
                continue

for i in range(len(track)):
    for j in range(len(track)):
        if [i, j] in get_start_states(track).tolist():
            try:
                plt.arrow(i , j, 1, 0,
                          width=0.5, head_width=0,
                          color="green",alpha=0.9)
            except IndexError:
                continue

for i in range(len(track)):
    for j in range(len(track)):
        if [i, j] in get_end_states(track).tolist():
            try:
                plt.arrow(i + 1, j, 0, 1,
                          width=0.5, head_width=0,
                          color="red",alpha=0.9)
            except IndexError:
                continue

plt.xlabel("X coordinate", size=20)
plt.ylabel("Y coordinate", size=20)
plt.tight_layout()
plt.show()
```


# Exercise 7, answers

## Problem 1

### Part a)

Let's assume the opposite that $x^*$ is an optimal solution to
$$
\min \sum_{i=1}^kw_if_i(x)\\
\text{s.t. }x\in S
$$
and that the solution $x^*$ is not Pareto optimal.

If $x^*$ is not Pareto optimal, then there exists $x'\in S$ such that
$$
\left\{
\begin{align}
f_i(x')\leq f_i(x^*) \text{ for all }i=1,\ldots,k,\text{ and}\\
f_j(x')< f_j(x^*) \text{ for some }j=1,\ldots,k.
\end{align}
\right.
$$
But this means that
$$
\begin{align}
\sum_{i=1}^kw_if_i(x') &= \sum_{i\neq j}w_if_i(x')+w_jf_j(x')\\
&<\sum_{i\neq j}w_if_i(x^*)+w_jf_j(x^*)\\
&=\sum_{i=1}^kw_if_i(x^*).
\end{align}
$$
Thus, $x^*$ is not an optimal solution to the problem in the weighting method. This means that $x^*$ must be Pareto optimal.

### Part b

Let's again assume that a solution $x^*$ is an optimal solution to the problem for all $j=1,\ldots,k$ and $\epsilon = f(x^*)$, but it is not Pareto optimal.

If $x^*$ is not Pareto optimal, then there exists $x'\in S$ such that
$$
\left\{
\begin{align}
f_i(x')\leq f_i(x^*) \text{ for all }i=1,\ldots,k,\text{ and}\\
f_j(x')< f_j(x^*) \text{ for some }j=1,\ldots,k.
\end{align}
\right.
$$

The first inequality means that $x'$ is a feasible solution to the epsilon constraint method for "$j=j$" (meaning that the index $j$ is the same in the definition of Pareto optimality and the definition of the epsilon constraint method).

Furthermore, the second stricth inequality says that the value of the objective at this solutions $x'$ is better than the value at the solutions $x^*$. This means that it is not Pareto optimal.

# Problem 2

# Part a)

Let's assume that a solution $x^*$ is an optimal solution to the achievement scalarizing problem, but it is not Pareto optimal.

If $x^*$ is not Pareto optimal, then there exists $x'\in S$ such that
$$
\left\{
\begin{align}
f_i(x')\leq f_i(x^*) \text{ for all }i=1,\ldots,k,\text{ and}\\
f_j(x')< f_j(x^*) \text{ for some }j=1,\ldots,k.
\end{align}
\right.
$$

Now,
$$
\max_{i=1}^i(f_i(x^*)-z^{ref}_i)+\rho\sum_{i=1}^kf_i(x^*) \geq \max_{i=1}^i(f_i(x')-z^{ref}_i)+\rho\sum_{i=1}^kf_i(x^*)\\
>  \max_{i=1}^i(f_i(x')-z^{ref}_i)+\rho\sum_{i=1}^kf_i(x')
$$
where we used just the first inequality first, and then we used the previous proof from weighting method.

This means again that if the solution $x^*$ is not Pareto optimal, then it cannot be an optimal solution to the achievement scalarizing function.

Part b)

Assume a Pareto optimal $x^*$. Now, we choose $z^{ref} = f(x^*)$. Let's have any solutions $x\in S$. Then,
$$
\max_{i=1\ldots,k}(f_i(x)-z_i^{ref})+\rho\sum_{i=1}^kf_i(x) = \max_{i=1\ldots,k}(f_i(x)-f_i(x^*))\\
\geq 0,
$$
because $x^*$ is Pareto optimal and, thus, at least one of the objectives needs to be greater or equal (if this was not the case, the solution $x^*$ would not be Pareto optimal).

But now,
$$
\max_{i=1\ldots,k}(f_i(x^*)-z_i^{ref})+\rho\sum_{i=1}^kf_i(x^*)= \max_{i=1\ldots,k}(f_i(x^*)-f_i(x^*))+\rho\sum_{i=1}^kf_i(x^*) \\
= 0
$$
trivially. This means that the Pareto optimal solution $x^*$ is an optimal solution to the achievement scalarizing problem.


Marec 2015, J.Slavič in L.Knez

Vprašanje 1: Za sistem enačb:
    $$
    \mathbf{A}=
    \begin{bmatrix}
    1 & -4 & 1\\
    1 & 6 & -1\\
    2 & -1 & 2
    \end{bmatrix}
    \qquad
    \mathbf{b}=
    \begin{bmatrix}
    7\\
    13\\
    5
    \end{bmatrix}
    $$

najdite rešitev s pomočjo ``SymPy``.

```python
from sympy import *
init_printing()

# Definiramo simbolične spremenljivke
x1, x2, x3 = symbols('x1, x2, x3')

# Definiramo sistem
A = Matrix([[1, -4, 1],
            [1, 6, -1],
            [2, -1, 2]])
x = Matrix([[x1],
            [x2],
            [x3]])
b = Matrix([[7],
            [13],
            [5]])

eq = Eq(A*x,b)
eq
```

```python
resitev = solve(eq,[x1, x2, x3])
resitev
```

Vprašanje 2: Za zgoraj definirano matriko $\mathbf{A}$ določite Evklidsko normo (lastni program).

```python
# Pretvorimo podatke v np.array

import numpy as np
a = np.array(A).astype(float)   # Pazite, da spremenite v float, v osnovi je object
a
```

    array([[ 1., -4.,  1.],
           [ 1.,  6., -1.],
           [ 2., -1.,  2.]])

```python
# Hitra varianta

A_evk = np.sqrt(np.sum(a**2))
A_evk
```

```python
# Pocasna varianta

vsota = 0
for vrstica in a:
    for element in vrstica:
        vsota += element**2

A_evk = vsota**0.5
A_evk

```

Vprašanje 3: Za zgoraj definirano matriko $\mathbf{A}$ določite neskončno normo (lastni program).

```python
# Hitra varianta

A_oo = max(abs(a).sum(axis=1))
A_oo
```

```python
# Pocasna varianta

vsota = zeros(1,3)
for i, vrstica in enumerate(a):
    vsota[i] = sum(abs(vrstica))

A_oo = max(vsota)
A_oo
```

Vprašanje 4: Za zgoraj definirano matriko $\mathbf{A}$ določite pogojenost (``numpy`` funkcija).

```python
pogojenost = np.linalg.cond(a)
pogojenost
```

Vprašanje 5: Definirajte funkcijo ``gauss_elim``, ki za poljubno matriko $\mathbf{A}$ in vektor $\mathbf{b}$ izvede Gaussovo eliminacijo (posebej za matriko in posebej za vektor).

```python
# Definiramo podatke
A_pod = np.array([[1, -4, 1],
                  [1, 6, -1],
                  [2, -1, 2]], dtype=float)

b_pod = np.array([[7],
                  [13],
                  [5]], dtype=float)
```

```python
# Funkcija
def gauss_elim(A, b):
    n = len(b)

    for k in range(0,n-1):
        for i in range(k+1,n):
            if A[i,k] != 0.0:
                faktor =  A[i,k]/A[k,k]
                A[i,k:n] = A[i,k:n] - faktor*A[k,k:n]
                b[i] = b[i] - faktor*b[k]

    return A, b

```

```python
# Uporaba funkcije
[A, b] = gauss_elim(A_pod.copy(), b_pod.copy())

A
```

    array([[  1. ,  -4. ,   1. ],
           [  0. ,  10. ,  -2. ],
           [  0. ,   0. ,   1.4]])

```python
b
```

    array([[  7. ],
           [  6. ],
           [-13.2]])

Vprašanje 6: Definirajte funkcijo ``gauss_elim_x``, ki za razultat funkcije ``gaus_elim`` najde ustrezne vrednosti vektorja $\textbf{x}$.

```python
# Funkcija
def gauss_elim_x(A, b):
    Ab = np.column_stack((A,b))   # Sestavimo v sistem Ab
    n = len(b)
    res = np.zeros(n)             # Pripravimo seznam ničel za rešitev

    for k, vrsta in enumerate(Ab[::-1]):
        res[n-k-1] = (vrsta[-1] - np.dot(vrsta[n-k:-1], res[n-k:]) ) / (vrsta[n-k-1])

    return res
```

```python
x = gauss_elim_x(A.copy(), b.copy())
x

```

    array([ 11.28571429,  -1.28571429,  -9.42857143])

```python
# Preverimo rešitev
ostanek = np.dot(A_pod,x) - b_pod.T
ostanek
```

    array([[ -8.88178420e-16,   0.00000000e+00,  -2.66453526e-15]])

```python

```

pypinm-master/vprasanja za razmislek/Vaja 5 - polovica.ipynb


```python
"""2.Phase"""

from IPython.display import display
from sympy import *

# -----------------------------------------------------------------
### Globlas ###
init_printing()
LIMIT = 100 # im Fall von kreisen
ein, ver = var('eintretende verlassende')

# -----------------------------------------------------------------
### LP ###
z, x1, x2, x3, x4, x5, x6, x7 = symbols("""
    z, x1, x2, x3, x4, x5, x6, x7""")

B = [x1, x2, x4, x6, x7]
N = [x3, x5]

rows = [Eq(x4, 6   + 3 * x5 - 1 * x3),
        Eq(x1, 2   -     x5 + 1 * x3),
        Eq(x2, 8   + 2 * x5 - 1 * x3),
        Eq(x6, 22  - 5 * x5 + 1 * x3),
        Eq(x7, 10  + 1 * x5 - 1 * x3)]
ziel = Eq(z,   86  + 5 * x5 + 3 * x3)

# show the given problem
print("given LP")
for row in rows:
    display(row)
display(ziel)
print("-"*30)

# -----------------------------------------------------------------
### Solving ###
for i in range(LIMIT):
    # eintretende Variable finden
    # auswaehlen nach dem Teknik in der Vorlesung
    # (d.h. var mit grosstem Koeffizeint)
    eintretende = None
    max_eintretende = -oo
    for var, coeff in ziel.rhs.as_coefficients_dict().items():
        # 1 is the first coeff i.e. the value of the ziel function
        if var != 1 and  coeff > 0 and coeff > max_eintretende:
                max_eintretende = coeff
                eintretende = var

    # falls keien positive costs => optimal
    if eintretende == None:
        break

    # verlassende Variable finden
    verlassende = None
    min_wert = +oo
    min_row = None

    for row in rows:
        if row.has(eintretende):
            new_row = row
            for nbv in N:
                if nbv != eintretende:
                    new_row = new_row.subs(nbv, 0)
            wert = solve(new_row.rhs >= 0).as_set().right
            if wert < min_wert:
                min_wert = wert
                min_row = row
                verlassende = row.lhs

    # display eintretende und verlassende Variablen
    print("Step ", i)
    if i == 0: print("Unzulaessiges Dict")
    display(Eq(ein, eintretende))
    display(Eq(ver, verlassende))

    # die Formlen umsetzen und rows updaten
    new_formel = Eq(eintretende, solve(min_row, eintretende)[0])
    new_rows = [new_formel]
    for row in rows:
        if row.lhs != verlassende:
            new_rows.append(
                Eq(row.lhs, row.rhs.subs(eintretende, new_formel.rhs))
            )
    rows = new_rows

    # new ziel
    ziel = Eq(z, ziel.rhs.subs(eintretende, new_formel.rhs))

    # update B, N
    B.remove(verlassende); B.append(eintretende)
    N.remove(eintretende); N.append(verlassende)

    # show current solution
    for row in rows:
        display(row)
    display(ziel)
    print("-"*30)
```

```python

```