```python
"""
In this script, the variable dimensions of our models are determined.
In the first part we calculate the piston sealing grooves.
Secondly, the dimensions of the non-circular shapes are determined.
"""

__author__ = "Eva Zillen"
__copyright__ = "Copyright 2021, TU Delft Biomechanical Design"
__credits__ = ["Eva Zillen, Heike Vallery, Gerwin Smit"]
__license__ = "CC0-1.0 License"
__version__ = "1.0.0"
__maintainer__ = "Eva Zillen"
__email__ = "e.zillen@student.tudelft.nl"
```

# Definitions

## Imports

```python
import numpy as np
import math
from sympy import symbols, Eq, solve, pi
```

## Functions

```python
def calculate_groove(ID,S,r_sq,C):
    # Determine the outer diameter of the O-ring
    OD = ID+2*S
    # Determine the squeeze dimension of the O-ring
    S_sq = S * (1 - r_sq)
    # Determine the pison groove diameter
    D_PG = OD - 2 * S_sq
    # Set the width of the piston groove
    W_PG = S + 1
    # Determine the piston diameter
    P_D = OD - 2 * C
    return f'\nOuter diameter O-ring: {OD} mm \nPiston diameter: {P_D} mm \nPiston groove diameter: {D_PG} mm \nWidth piston groove: {round(W_PG,2)} mm\n'
```

# Determining the dimensions of the O-ring groove

#### O-ring Lidl, SLA print

```python
# The O-ring used had the dimensions (18x3.5)
ID = 18
S = 3.5
# A clearance of 0.5 mm was used between the piston and the cylinder
C = 0.5
# 10% squeeze ration was used in this study
r_sq = 0.1

print ("O-ring Lidl, 18*3.5")
print(calculate_groove(ID,S,r_sq,C))
```

    O-ring Lidl, 18*3.5

    Outer diameter O-ring: 25.0 mm
    Piston diameter: 24.0 mm
    Piston groove diameter: 18.7 mm
    Width piston groove: 4.5 mm

#### O-ring en X-ring Eriks

```python
# The O-ring used had the dimensions (18.64x3.53)
ID = 18.64
S = 3.53
# A clearance of 0.5 mm was used between the piston and the cylinder
C = 0.5
# 10% squeeze ration was used in this study
r_sq = 0.1

print ("O-ring en X-ring Eriks, 18.64*3.53")
print(calculate_groove(ID,S,r_sq,C))
```

    O-ring en X-ring Eriks, 18.64*3.53

    Outer diameter O-ring: 25.7 mm
    Piston diameter: 24.7 mm
    Piston groove diameter: 19.346 mm
    Width piston groove: 4.53 mm

# Determining the dimensions of the non-conventional cylinders

### Stadium shape

```python
def stadium(ID,S):
    # Variables defining the stadium shape (see Figure 2 in report)
    L = symbols('L')
    D = 0

    # Target area - the area of the circular shape with diameter = 25 mm
    A_target = math.pi * (25/2)**2
    # Determine the outer diameter of the O-ring
    OD = ID + 2 * S
    # Determine the perimeter of the O-ring
    P_c = math.pi * OD

    # Variable necessary to perform the optimization
    A_s = 0

    # To find the variables D & L we use for the stadium surface area (A_s) and the perimeter (P_s) equations
    # Optimization is done by varying D and finding corresponding L
    for D_optimize in np.arange(5,20,0.1):
        # Variables necessary for optimization, to remember the previous values
        previous_A_s = A_s
        previous_L = L
        previous_D = D

        # L is reset each round, define it again
        L = symbols('L')

        # Rounding necessary for limitations on floating point numbers in Python
        D = round(D_optimize,2)

        # Solve for the perimeter
        P_s_eq = (math.pi * D) + 2 * L - P_c
        result = solve((P_s_eq), (L))

        # Solving returns a list, L is the only value in this list
        L = result[0]

        # Applying the equation for the surface area
        A_s = math.pi * (D/2)**2 + L * D

        # Finding the right value: if the surface area is larger than the target area, take the previous and continue
        if previous_A_s <= A_target and A_s >= A_target:
            print(f"Optimizing found: \narea = {previous_A_s} with D = {previous_D} and L = {previous_L} is closest to the target surface area of {A_target}")
            break

    return previous_D,previous_L

```

```python
# Example calculations for an O-ring of 22x3.5mm
ID = 22
S = 3.5
D, L = stadium(ID,S)
print(f"\n--------------------------------------")
print(f"Resulting values for an stadium shape with O-ring of {ID} x {S} mm")
print(f"D = {D} mm")
print(f"L = {L} mm")
print(f"--------------------------------------")
```

    Optimizing found:
    area = 490.803166288700 with D = 14.3 and L = 23.0907060038850 is closest to the target surface area of 490.8738521234052

    --------------------------------------
    Resulting values for an stadium shape with O-ring of 22 x 3.5 mm
    D = 14.3 mm
    L = 23.0907060038850 mm
    --------------------------------------

### Kidney shape

```python
def optimize_range(ID, S, gamma):
    # Variables defining the kidney shape (see Figure 2 in report)
    r = symbols('r')
    a = 0

    # Target area - the area of the circular shape with diameter = 25 mm
    A_target = math.pi*(25/2)**2
    # Determine the outer diameter of the O-ring
    OD = ID+2*S
    # Determine the perimeter of the O-ring
    P_c = math.pi*OD

    # Variable necessary to perform the optimization
    A_k = 0

    # To find the variables r & a we use the stadium surface area (A_k) and the perimeter (P_k) equations
    # Optimization is done by varying a and finding corresponding r
    for a_optimize in np.arange(5,20,0.1):
        # Variables necessary for optimization, to remember the previous values
        previous_A_k = A_k
        previous_r = r
        previous_a = a

        # L is reset each round, define it again
        r = symbols('r')

        # Rounding necessary for limitations on floating point numbers in Python
        a = round(a_optimize,2)

        # Solve for the perimeter
        P_k_eq = math.pi* a +(gamma*(2 * r + a)) - P_c
        result = solve((P_k_eq), (r))

        # Solving returns a list, L is the only value in this list
        r = result[0]

        # Applying the equation for the surface area
        A_k = math.pi * (a / 2)**2 + (gamma * (r * a + (a**2 / 2)))

        # Finding the right value: if the surface area is larger than the target area, take the previous and continue
        if previous_A_k <= A_target and A_k >= A_target:
            print(f"Optimizing found:\narea = {previous_A_k} with a = {previous_a} and r = {previous_r} is closest to the target surface area of {A_target}")
            break

    return previous_a,previous_r
```

```python
# Example calculations for an O-ring of 22x3.5mm
ID = 22
S = 3.5
gamma = 100/180*math.pi

a, r = optimize_range(ID, S, gamma)
print(f"\n--------------------------------------")
print(f"Resulting values for a kidney shape with O-ring of {ID} x {S} mm")
print(f"a = {a} mm")
print(f"r = {r} mm")
print(f"--------------------------------------")
```

    Optimizing found:
    area = 490.803166288700 with a = 14.3 and r = 6.08000000000001 is closest to the target surface area of 490.8738521234052

    --------------------------------------
    Resulting values for a kidney shape with O-ring of 22 x 3.5 mm
    a = 14.3 mm
    r = 6.08000000000001 mm
    --------------------------------------


## just some checks on ideal_gas_lumped.py and ideal_gas_distributed.py

these two modules are used to
insert equations (sympy expressions)
which model an ideal gas
into dictionnaries

```python
import sympy
sympy.init_printing()
```

```python
import sys
sys.path.insert(0, './code')

from symbolic import eval_expr

import ideal_gas_lumped as lgas
import ideal_gas_distributed as dgas
```

```python
ltmp = {
    lgas._m_a: lgas.m_Ar,
}
```

```python
dtmp = {
    dgas._m_a: dgas.m_Ar,
}
```

```python
lstate = {
    lgas._s: 2.3394613409617664,
    lgas._v: 0.0003691371367968008,
    lgas._m: 0.000605034917759643,
}
```

```python
dstate = {
    dgas._s: 2.3394613409617664 / 0.0003691371367968008,
    dgas._m: 0.000605034917759643 / 0.0003691371367968008,
}
```

```python
# U(s, v, m)
u_lgas = eval_expr(lgas._U, {**lgas._functionals, **ltmp}, lgas.params, lstate).evalf()
u_lgas
```

```python
# U(s, m)
u_dgas = eval_expr(dgas._U, {**dgas.functionals, **dtmp}, dgas.params, dstate).evalf()
u_dgas * 0.0003691371367968008
```

```python
# U(s, m)
u = eval_expr(dgas._U, functionals=dgas.functionals)
u
```

```python
θ = eval_expr(dgas._θ, functionals=dgas.functionals)
θ
```

```python
μ = eval_expr(dgas._μ, functionals=dgas.functionals)
μ
```

```python
π = eval_expr(dgas._π, functionals=dgas.functionals)
π
```


$\newcommand{\ket}[1]{\left|{#1}\right\rangle} \newcommand{\bra}[1]{\left\langle{#1}\right|}$
## Tehtävä 1: Blochin pallo

Blochin pallo on tapa hahmottaa yhden kubitin kvanttisysteemin tilaa kolmiulotteisella pallolla.

Puhtaan kvanttitilan, eli itsenäisesti tarkasteltavan eristetyn yhden kubitin kvanttitilan Bloch-vektori on aina pallon pinnalla. (Puhdas tila ei siis tarkoita pelkästään $\ket 0$- ja $\ket 1$-tiloja)

Yksi tapa määrittää kvanttitilaa vastaava piste tai vektori Blochin pallolla on valita kuvan kulmat $\theta$ ja $\phi$ siten, että ne vastaavat kubitin tilaa $\ket \psi$ seuraavasti:

$$\ket \psi = cos(\theta / 2)\ket 0 + e^{i\phi}sin(\theta / 2)\ket 1$$

Bloch-vektorin ja z-akselin välinen kulma $\theta$ kertoo siis $\ket 0$-kertoimen ja $\ket 1$-kertoimen itseisarvojen välisestä suhteesta. Kuvaan merkitäänkin usein pisteet, joissa tila on kokonaan $\ket 0$ (ylhäällä) tai $\ket 1$ (alhaalla). Toisaalta Bloch-vektorin ja x-akselin välinen kulma $\phi$ kuvastaa $\ket 1$:n amplitudin vaihetta eli suuntaa.

Tilavektorin $\ket 0$-komponentin kerroin on sovitusti aina positiivinen reaaliluku. Tämä voidaan tehdä siksi, että systeemin ns. globaali vaihe ei vaikuta mittaustuloksiin, myöskään unitaaristen porttien käytön jälkeen.

Unitaarinen portti on Blochin pallolla aina jokin kierto tietyn akselin ympäri tietyn kulman verran. Esimerkkejä tavallisten porttien operaatioista Blochin pallon kääntöinä:

 - $X$: Kääntää x-akselin ympäri kulman $\pi = 180^o$ verran. Käännön suunnalla ei ole merkitystä, koska puolikas kierto tuottaa saman tuloksen molempiin suuntiin.
 - $Y$: Kääntää y-akselin ympäri kulman $\pi$ verran.
 - $Z$: Kääntää z-akselin ympäri kulman $\pi$ verran.
 - $H$: Kääntää x+z-akselin (eli puolivälin x ja z -akselien välillä) ympäri kulman $\pi$ verran.
 - $S$: Kääntää z-akselin ympäri kulman $\pi/2$ verran, vastapäivään ylhäältä katsottuna.
 - $T$: Kääntää z-akselin ympäri kulman $\pi/4$ verran, jälleen vastapäivään ylhäältä katsottuna.

Demo Blochin pallon toiminnasta: https://bloch.ollpu.fi/

Näkymää voi kääntää vetämällä palloa hiirellä ja zoom toimii myös. Kuvassa akselit $x, y, z$ piirretään punaisella, vihreällä ja sinisellä.

Systeemin alkutila on $\ket 0$. Voit käyttää portteja oikealla olevista napeista. Kokeile kaikkien porttien toimintaa, ja totea, että ne toimivat kuten yllä on kuvailtu. Kiinnitä huomiota myös siihen, miten portit vaikuttavat tilavektoriin, joka esitetään näkymän yläpuolella. Siinä $\ket 0$:n amplitudi voi olla myös kompleksiluku, toisin kuin aiemmin sovittiin, mutta amplitudeista voidaan laventaa yhteinen tekijä pois.

Tämän viikon luennolla puhuttiin aikakehityksestä. Luennolla esitetty Hamiltoninen aikakehitys vastaa Blochin pallolla jatkuvaa pyöritystä X-akselin ympäri myötäpäivään. Voit tarkastella sen vaikutusta tilaan "Simuloi Hamiltonista aikakehitystä"-napista.

## Tehtävä 2: Z-portin toteus kvanttitietokoneesa

### Johdanto
Aiemmin kubittien tilat $\ket +$ ja $\ket -$ määritettiin seuraavasti:
$$
 \ket + = \frac{1}{\sqrt 2} (\ket 0 +  \ket 1), \\
 \ket - = \frac{1}{\sqrt 2} (\ket 0 -  \ket 1).
$$

Z-portin (ns. phase-flip gate) matriisiesitys on seuraava:

$$
Z=  \pmatrix {1 & 0 \\ 0 & -1}
$$

Z-portin vaikutus esimerkiksi tiloihin $\ket +$ ja $\ket -$ on seuraava:

$$
\begin{align}
Z\ket + &= \ket - \text{ja}\\
Z\ket - &= \ket +
\end{align}
$$

### Tehtävä
Professori Mikko Möttöinen kertoi luennolla, ettei kvanttitietokoneen valmistajan tarvitse realisoida kaikkia kvanttitietokoneen portteja, koska useiden porttien toiminta on korvattavissa muilla porteilla. Miten Z-portin operaatio on toteutettavissa Hadamard H-portilla ja X-portilla eli not-portilla?

Ohje: Tutki 1. tehtävän demolla kuinka Z-portti kiertää Blochin pallolle piirrettyä tilavektoria, tai päättele miten Z-portin matriisiesitys saadaan tulona H- ja X-porttien matriisiesityksistä. Kertauksena:

$$
X = \pmatrix {0 & 1 \\ 1 & 0} \quad \text{ja} \quad H=\frac 1 {\sqrt{2}}\pmatrix {1 & 1 \\ 1 & -1}
$$

## Tehtävä 3:  Piirin tilavektorin tarkastelua

### Johdanto
Seuraavassa esimerkissä luodaan yhden kubitin kvanttipiiri, ja tulostetaan piirin tilavektori ja sen esitys Blochin pallolla. Koodin tulosteessa merkintä `1+0j` tarkoittaa, että kompleksiluvun reaaliosa on 1 ja imaginaariosa 0, ja  `j` merkitsee imaginääriyksikköä. Koodissa tulostuksen muotoiluun käytettävää `enumerate(statevector)`-syntaksia ei tällä kurssilla tarvitse ymmärtää.

```python
from qiskit import *

qreg = QuantumRegister(1)
creg = ClassicalRegister(1)
circuit = QuantumCircuit(qreg, creg)
circuit.draw(output='mpl')
```

```python
simulator = Aer.get_backend('statevector_simulator')

# Suoritetaan
result = execute(circuit, simulator).result()
statevector = result.get_statevector(circuit)

# Tulostetaan saatu tila ket-notaation kera
for i, c in enumerate(statevector):
    print(c, f"|{i:0b}>")

visualization.plot_bloch_multivector(statevector, title="Alkutila")
```

**a)** Ohjelmoi yllä olevan esimerkin mukaisesti yhden kubitin kvanttipiiri, jossa kubitin tila on alussa $\ket 1$, jonka jälkeen kubittiin operoidaan H-portilla. Simuloi piiri, ja tulosta tämän jälkeen tilavektori käyttäen komentoa `result.get_statevector(circuit)` ja piirrä yhden kubitin systeemin tilavektori Blochin pallolla.

Tuloksen pitäisi näyttää jotakuinkin seuraavalta:

Huomaa, tulosteessa, että kompleksiluvun kerroin $1 / \sqrt 2 \approx 0.707$ ja että imagarinääriosa $10^{-17} \approx 0$. Numeerisesta virheestä johtuen tuloksen imaginääriosa ei ole täsmälleen nolla, mutta äärimmäisen lähellä nollaa. Kuvan tila on siis saavutettu operoimalla tilaan $\ket 1$ portilla H:

$$
H\ket 1 = \frac 1 {\sqrt{2}}(\ket 0 - \ket 1) = \ket -
$$

Blochin pallo sopii havainnollistamisvälineeksi yhden kubitin systemeissä.

```python
# kirjoita vastauksesi tähän

```

**b)** Jatka a-osaa ja tutki simuloimalla, että $H \ket - = \ket 1$. Esitä ratkaisuna lopputuloksen tilavektori ja sen esitys Blochin pallolla.

```python
# ohjelmoi ratkaisusi tähän

```

**c)** Tarkastellaan johdantona seuraavaa kahden kubitin kvanttipiriä.

Systeemin alkutila on $\ket{00}$.

Portin vaikutus tilaan voidaan aina laskea superposition jokaiselle osalle, eli summan jokaiselle termille erikseen.

Yhden kubitin kääntöportin vaikutus syteemin tilaan voidaan määrittää sijoittamalla portin tulos kubitin osuuteen tensoritulossa.

Systeemin tila H-portin jälkeen:
$$
\ket 0 \otimes \frac{1}{\sqrt 2} (\ket 0 + \ket 1) = \frac{1}{\sqrt 2}(\ket{00}+\ket{01})
$$

Systeemin tila CNOT-portin jälkeen:
$$
\frac{1}{\sqrt 2}(\ket{00}+\ket{11})
$$

Systeemin tila X-portin jälkeen:
$$
\frac{1}{\sqrt 2}(\ket{10}+\ket{01})
$$

Alla piirin rakentamisessa käytetty koodi:

```python
from qiskit import *

qreg = QuantumRegister(2)
creg = ClassicalRegister(2)
circuit = QuantumCircuit(qreg, creg)
circuit.h(qreg[0])
circuit.cx(qreg[0],qreg[1])
circuit.x(qreg[0])
circuit.draw(output='mpl')
```

Simuloidaan piirin toiminta ja tulostetaan tilavektori:

```python
akselin ympäri tietyn kulman verran. Esimerkkejä tavallisten porttien operaatioista Blochin pallon kääntöinä:simulator = Aer.get_backend('statevector_simulator')

# Suoritetaan
result = execute(circuit, simulator).result()
statevector = result.get_statevector(circuit)
# Tulostetaan saatu tila ket-notaation kera
for i, c in enumerate(statevector):
    print(c, f"|{i:02b}>")

```

    0j |00>
    (0.7071067811865476+0j) |01>
    (0.7071067811865475+0j) |10>
    0j |11>

**Tehtävä.** Simuloi kuvan piirin toiminta ja tulosta tilavektori:

Perustele saamasi tulos laskemalla ket-esitystä käyttäen.

```python
# akselin ympäri tietyn kulman verran. Esimerkkejä tavallisten porttien operaatioista Blochin pallon kääntöinä:kirjoita vastauksesi tähän

```

## Tehtävä 4

Luennolla käsiteltiin Fourier-muunnosta. Katso halutessasi havainnollistava video siihen liittyen (linkki löytyy myös Classroomista):

https://www.youtube.com/watch?v=spUNpyF58BY

```python

```


## Open a dataset

```python
import pandas as pd

fn = "../data/benchmarks/diffeq/predpreyfrac_clean.csv"
df = pd.read_csv(fn, skipinitialspace=True)
print df.columns
```

    Index([u'T', u'x', u'y', u'dx', u'dy'], dtype='object')

## Graph the data

```python
# visualization libraries
import matplotlib.pyplot as plt
# plot the visuals in ipython
%matplotlib inline

df[["x","y"]].plot()
df[["dx","dy"]].plot()
print ""
```

```python
from pypge import (evaluate, model)
import sympy

X_train = df[["x", "y"]].as_matrix().T
Y_train = df[["dx"]].as_matrix().reshape(5000,)

print X_train.shape, Y_train.shape

xs = sympy.symbols("x y")

eqn_str = '-0.2*x + 0.001*x*y'
# eqn_str = "C*x + C*x*y"
eqn = sympy.sympify(eqn_str)

modl = model.Model(eqn, xs)

y_pred = evaluate.Eval(modl, xs, X_train)
print y_pred.shape, Y_train.shape

mae, err = evaluate.Score(Y_train, y_pred, "rmse")
print mae, err

```

    (2, 5000) (5000,)
    (5000,) (5000,)
    0.0198915007578 None


```python
import matplotlib.pyplot as plt
import scipy.stats as st
import seaborn as sns
import pandas as pd
from scipy.stats import norm, uniform, expon, t, binom, binom_test, wilcoxon
from scipy.integrate import quad
from sympy.solvers import solve
from sympy import Symbol
import numpy as np
from pandas import Series, DataFrame
```

```python
fuellungen = Series([71, 69, 67, 68, 73, 72, 71, 71, 68, 72, 69, 72])
```

```python
fuellungen.median()
```

    71.0

```python
alleGroesseren = fuellungen >= fuellungen.median()
```

```python
len(fuellungen[alleGroesseren])
```

    7

```python
binom.cdf(k=7, n=fuellungen.size, p=0.5)
```

    0.80615234375

```python
1-st.binom_test(x=5, n=fuellungen.size, p=0.5, alternative="less")
```

    0.6127929687499998

```python
binom.ppf(q=[0.95], n=fuellungen.size, p=0.5)
```

    array([9.])

```python
binom.ppf(q=[0.05, 0.975], n=12, p=0.5)
```

    array([3., 9.])

```python
binom.cdf(k=2, n=12, p=0.5)
```

    0.019287109375000007

```python
2,8
```

    (2, 8)

```python
wilcoxon(np.array([71, 69, 67, 68, 73, 72, 71, 71, 68, 72, 69, 72])-70, zero_method="wilcox", correction=True)
```

    WilcoxonResult(statistic=33.5, pvalue=0.6902117434795202)

```python
serie1 = Series([120, 265, 157, 187, 219, 288, 156, 205, 163])
serie2 = Series([127, 281, 160, 185, 220, 298, 167, 203, 171])
```

    Ttest_indResult(statistic=-0.22026187957798735, pvalue=0.8284564724086932)

```python
print(serie1.mean())
print(serie1.std())
```

    195.55555555555554
    54.65828187729448

```python
print(serie2.mean())
print(serie2.std())
```

    201.33333333333334
    56.61492736019362

```python
t.ppf(q=0.05, loc=0, scale=6.2/np.sqrt(serie1.size), df=serie1.size-1)
```

    -3.8430659442138757

# 8.4

```python
jackals = pd.read_table(r"jackals.txt", sep = " ")
jackals
st.ttest_ind(jackals["M"], jackals["W"], equal_var=False)
```

    Ttest_indResult(statistic=3.4843242131699643, pvalue=0.0033599524351925016)

```python
st.mannwhitneyu(jackals["M"], jackals["W"], alternative = "two-sided")
```

    MannwhitneyuResult(statistic=87.5, pvalue=0.004845462884722891)

```python
Zurich = Series([16.3, 12.7, 14.0, 53.3 ,117, 62.6 ,27.6])
Basel = Series([10.4, 8.91, 11.7, 29.9 ,46.3, 25.0 ,29.4])
```

```python
differences = np.abs(Zurich - Basel)
```

```python
differences.mean()
```

    20.784285714285716

```python
differences.std()
```

    25.79925183485163

```python
st.ttest_ind(Zurich, Basel, equal_var=False)
```

    Ttest_indResult(statistic=1.3273296255450922, pvalue=0.2232566584677775)

```python
t.interval(alpha=0.95, df=differences.size, loc=differences.mean(), scale=differences.std()/np.sqrt(differences.size))
```

    (-2.273639757199689, 43.84221118577112)

```python

```


# ***Introduction to Radar Using Python and MATLAB***
## Andy Harrison - Copyright (C) 2019 Artech House
<br/>

# Single Pulse Ambiguity Function
***

Referring to Section 8.6.1, the complex envelope of a single unmodulated pulse is expressed as (Equation 8.59)

\begin{equation}\label{eq:single_unmodulated_pulse}
    s(t) = \text{rect}\bigg[ \frac{t}{\tau_p} \bigg],
\end{equation}

\noindent Using (Equation 8.59) and (Equation 8.53) allows the ambiguity function to be written as (Equation 8.61)

\begin{equation}
    \big| \chi(\tau, f) \big|^2  = \Bigg| \int\limits_{-\infty}^{\infty} \text{rect}\bigg[ \frac{t}{\tau_p}\bigg] \, \text{rect}\bigg[ \frac{t - \tau}{\tau_p} \bigg] e^{j 2 \pi f t}\, dt\Bigg|^2.
\end{equation}

Carrying out the integration gives (Equation 8.62)

\begin{equation}
    \big| \chi(\tau, f) \big|^2  = (\tau_p - |\tau|)^2\, \Big|\text{sinc}\left[ f (\tau_p - |\tau|) \right]  \Big|^2 \quad \text{for  } |\tau| \le \tau_p.
\end{equation}
***

Begin by getting the library path

```python
import lib_path
```

Set the pulsewidth (s)

```python
pulsewidth = 1e-3
```

Generate the time delay (s) using the `linspace` routine from `scipy`

```python
from numpy import linspace

# Set the time delay

time_delay = linspace(-pulsewidth, pulsewidth, 5000)
```

Calculate the ambiguity function for the single pulse

```python
from Libs.ambiguity.ambiguity_function import single_pulse

from numpy import finfo

ambiguity = single_pulse(time_delay, finfo(float).eps, pulsewidth)
```

Plot the zero-Doppler cut using the `matplotlib` routines

```python
from matplotlib import pyplot as plt

# Set the figure size

plt.rcParams["figure.figsize"] = (15, 10)

# Plot the ambiguity function

plt.plot(time_delay, ambiguity, '')

# Set the x and y axis labels

plt.xlabel("Time (s)", size=12)

plt.ylabel("Relative Amplitude", size=12)

# Turn on the grid

plt.grid(linestyle=':', linewidth=0.5)

# Set the plot title and labels

plt.title('Single Pulse Ambiguity Function', size=14)

# Set the tick label size

plt.tick_params(labelsize=12)
```

Set the Doppler mismatch frequencies using the `linspace` routine

```python
doppler_frequency = linspace(-2.0 / pulsewidth, 2.0 / pulsewidth, 1000)
```

Calculate the ambiguity function for the single pulse

```python
ambiguity = single_pulse(finfo(float).eps, doppler_frequency, pulsewidth)
```

Display the zero-range cut for the single pulse

```python
plt.plot(doppler_frequency, ambiguity, '')

# Set the x and y axis labels

plt.xlabel("Doppler (Hz)", size=12)

plt.ylabel("Relative Amplitude", size=12)

# Turn on the grid

plt.grid(linestyle=':', linewidth=0.5)

# Set the plot title and labels

plt.title('Single Pulse Ambiguity Function', size=14)

# Set the tick label size

plt.tick_params(labelsize=12)
```

Set the time delay and Doppler mismatch frequency and create the two-dimensional grid using the `meshgrid` routine from `scipy`

```python
from numpy import meshgrid

# Set the time delay

time_delay = linspace(-pulsewidth, pulsewidth, 1000)

# Set the Doppler mismatch

doppler_frequency =linspace(-2.0 / pulsewidth, 2.0 / pulsewidth, 1000)

# Create the grid

t, f = meshgrid(time_delay, doppler_frequency)
```

Calculate the ambiguity function for the single pulse

```python
ambiguity = single_pulse(t, f, pulsewidth)
```

Display the two-dimensional contour plot for the single pulse ambiguity function

```python
# Plot the ambiguity function

plt.contour(t, f, ambiguity, 20, cmap='jet', vmin=-0.2, vmax=1.0)

# Set the x and y axis labels

plt.xlabel("Time (s)", size=12)

plt.ylabel("Doppler (Hz)", size=12)

# Turn on the grid

plt.grid(linestyle=':', linewidth=0.5)

# Set the plot title and labels

plt.title('Single Pulse Ambiguity Function', size=14)

# Set the tick label size

plt.tick_params(labelsize=12)
```

```python

```


# Basic (Gaussian likelihood) GP regression model

This notebook shows the different steps for creating and using a standard GP regression model, including:
  - reading and formatting data
  - choosing a kernel function
  - choosing a mean function (optional)
  - creating the model
  - viewing, getting, and setting model parameters
  - optimizing the model parameters
  - making predictions

We focus here on the implementation of the models in GPflow; for more intuition on these models, see [A Practical Guide to Gaussian Processes](https://drafts.distill.pub/gp/) and [A Visual Exploration of Gaussian Processes](https://distill.pub/2019/visual-exploration-gaussian-processes/).

```python
import gpflow
import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
from gpflow.utilities import print_summary

# The lines below are specific to the notebook format
%matplotlib inline
plt.rcParams["figure.figsize"] = (12, 6)
```

`X` and `Y` denote the input and output values. **NOTE:** `X` and `Y` must be two-dimensional NumPy arrays, $N \times 1$ or $N \times D$, where $D$ is the number of input dimensions/features, with the same number of rows as $N$ (one for each data point):

```python
data = np.genfromtxt("data/regression_1D.csv", delimiter=",")
X = data[:, 0].reshape(-1, 1)
Y = data[:, 1].reshape(-1, 1)

_ = plt.plot(X, Y, "kx", mew=2)
```

We will consider the following probabilistic model:
\begin{equation}
Y_i = f(X_i) + \varepsilon_i\,,
\end{equation}
where $f \sim \mathcal{GP}(\mu(\cdot), k(\cdot, \cdot'))$, and $\varepsilon \sim \mathcal{N}(0, \tau^2 I)$.

## Choose a kernel
Several kernels (covariance functions) are implemented in GPflow. You can easily combine them to create new ones (see [Manipulating kernels](../advanced/kernels.ipynb)). You can also implement new covariance functions, as shown in the [Kernel design](../tailor/kernel_design.ipynb) notebook. Here, we will use a simple one:

```python
k = gpflow.kernels.Matern52()
```

For more advanced kernels see the [advanced kernel notebook](../advanced/kernels.ipynb) (including kernels defined on subspaces). A summary of the kernel can be obtained by

```python
print_summary(k)
```

    ╒═══════════════════════╤═══════════╤═════════════╤═════════╤═════════════╤═════════╤═════════╤═════════╕
    │ name                  │ class     │ transform   │ prior   │ trainable   │ shape   │ dtype   │   value │
    ╞═══════════════════════╪═══════════╪═════════════╪═════════╪═════════════╪═════════╪═════════╪═════════╡
    │ Matern52.variance     │ Parameter │ Softplus    │         │ True        │ ()      │ float64 │       1 │
    ├───────────────────────┼───────────┼─────────────┼─────────┼─────────────┼─────────┼─────────┼─────────┤
    │ Matern52.lengthscales │ Parameter │ Softplus    │         │ True        │ ()      │ float64 │       1 │
    ╘═══════════════════════╧═══════════╧═════════════╧═════════╧═════════════╧═════════╧═════════╧═════════╛

The Matern 5/2 kernel has two parameters: `lengthscales`, which encodes the "wiggliness" of the GP, and `variance`, which tunes the amplitude. They are both set to 1.0 as the default value. For more details on the meaning of the other columns, see [Manipulating kernels](../advanced/kernels.ipynb).

## Choose a mean function (optional)
It is common to choose $\mu = 0$, which is the GPflow default.

However, if there is a clear pattern (such as a mean value of `Y` that is far away from 0, or a linear trend in the data), mean functions can  be beneficial. Some simple ones are provided in the `gpflow.mean_functions` module.

Here's how to define a linear mean function:

`meanf = gpflow.mean_functions.Linear()`

## Construct a model
A GPflow model is created by instantiating one of the GPflow model classes, in this case GPR. We'll make a kernel `k` and instantiate a GPR object using the generated data and the kernel. We'll also set the variance of the likelihood to a sensible initial guess.

```python
m = gpflow.models.GPR(data=(X, Y), kernel=k, mean_function=None)
```

A summary of the model can be obtained by

```python
print_summary(m)
```

    ╒═════════════════════════╤═══════════╤══════════════════╤═════════╤═════════════╤═════════╤═════════╤═════════╕
    │ name                    │ class     │ transform        │ prior   │ trainable   │ shape   │ dtype   │   value │
    ╞═════════════════════════╪═══════════╪══════════════════╪═════════╪═════════════╪═════════╪═════════╪═════════╡
    │ GPR.kernel.variance     │ Parameter │ Softplus         │         │ True        │ ()      │ float64 │       1 │
    ├─────────────────────────┼───────────┼──────────────────┼─────────┼─────────────┼─────────┼─────────┼─────────┤
    │ GPR.kernel.lengthscales │ Parameter │ Softplus         │         │ True        │ ()      │ float64 │       1 │
    ├─────────────────────────┼───────────┼──────────────────┼─────────┼─────────────┼─────────┼─────────┼─────────┤
    │ GPR.likelihood.variance │ Parameter │ Softplus + Shift │         │ True        │ ()      │ float64 │       1 │
    ╘═════════════════════════╧═══════════╧══════════════════╧═════════╧═════════════╧═════════╧═════════╧═════════╛

The first two lines correspond to the kernel parameters, and the third one gives the likelihood parameter (the noise variance $\tau^2$ in our model).

You can access those values and manually set them to sensible initial guesses. For example:

```python
m.likelihood.variance.assign(0.01)
m.kernel.lengthscales.assign(0.3)
```

    <tf.Variable 'UnreadVariable' shape=() dtype=float64, numpy=-1.0502256128148466>

## Optimize the model parameters

To obtain meaningful predictions, you need to tune the model parameters (that is, the parameters of the kernel, the likelihood, and the mean function if applicable) to the data at hand.

There are several optimizers available in GPflow. Here we use the `Scipy` optimizer, which by default implements the L-BFGS-B algorithm. (You can select other algorithms by using the `method=` keyword argument to its `minimize` method; see [the SciPy documentation](https://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.minimize.html) for details of available options.)

```python
opt = gpflow.optimizers.Scipy()
```

In order to train the model, we need to maximize the log marginal likelihood.
GPflow models define a `training_loss` that can be passed to the `minimize`
method of an optimizer; in this case it is simply the negative log marginal
likelihood. We also need to specify the variables to train with
`m.trainable_variables`, and the number of iterations.

```python
opt_logs = opt.minimize(m.training_loss, m.trainable_variables, options=dict(maxiter=100))
print_summary(m)
```

    ╒═════════════════════════╤═══════════╤══════════════════╤═════════╤═════════════╤═════════╤═════════╤═══════════╕
    │ name                    │ class     │ transform        │ prior   │ trainable   │ shape   │ dtype   │     value │
    ╞═════════════════════════╪═══════════╪══════════════════╪═════════╪═════════════╪═════════╪═════════╪═══════════╡
    │ GPR.kernel.variance     │ Parameter │ Softplus         │         │ True        │ ()      │ float64 │ 7.96581   │
    ├─────────────────────────┼───────────┼──────────────────┼─────────┼─────────────┼─────────┼─────────┼───────────┤
    │ GPR.kernel.lengthscales │ Parameter │ Softplus         │         │ True        │ ()      │ float64 │ 0.212416  │
    ├─────────────────────────┼───────────┼──────────────────┼─────────┼─────────────┼─────────┼─────────┼───────────┤
    │ GPR.likelihood.variance │ Parameter │ Softplus + Shift │         │ True        │ ()      │ float64 │ 0.0057594 │
    ╘═════════════════════════╧═══════════╧══════════════════╧═════════╧═════════════╧═════════╧═════════╧═══════════╛

Notice how the value column has changed.

The local optimum found by Maximum Likelihood might not be the one you want (for example, it might be overfitting or oversmooth). This depends on the initial values of the hyperparameters, and is specific to each dataset. As an alternative to Maximum Likelihood, [Markov Chain Monte Carlo (MCMC)](../advanced/mcmc.ipynb) is also available.

## Make predictions

We can now use the model to make some predictions at the new points `Xnew`. You might be interested in predicting two different quantities: the latent function values `f(Xnew)` (the denoised signal), or the values of new observations `y(Xnew)` (signal + noise). Because we are dealing with Gaussian probabilistic models, the predictions typically produce a mean and variance as output. Alternatively, you can obtain samples of `f(Xnew)` or the log density of the new data points `(Xnew, Ynew)`.

GPflow models have several prediction methods:

 - `m.predict_f` returns the mean and marginal variance of $f$ at the points `Xnew`.

 - `m.predict_f` with argument `full_cov=True` returns the mean and the full covariance matrix of $f$ at the points `Xnew`.

 - `m.predict_f_samples` returns samples of the latent function.

 - `m.predict_y` returns the mean and variance of a new data point (that is, it includes the noise variance).

 - `m.predict_log_density` returns the log density of the observations `Ynew` at `Xnew`.

We use `predict_f` and `predict_f_samples` to plot 95% confidence intervals and samples from the posterior distribution.

```python
## generate test points for prediction
xx = np.linspace(-0.1, 1.1, 100).reshape(100, 1)  # test points must be of shape (N, D)

## predict mean and variance of latent GP at test points
mean, var = m.predict_f(xx)

## generate 10 samples from posterior
tf.random.set_seed(1)  # for reproducibility
samples = m.predict_f_samples(xx, 10)  # shape (10, 100, 1)

## plot
plt.figure(figsize=(12, 6))
plt.plot(X, Y, "kx", mew=2)
plt.plot(xx, mean, "C0", lw=2)
plt.fill_between(
    xx[:, 0],
    mean[:, 0] - 1.96 * np.sqrt(var[:, 0]),
    mean[:, 0] + 1.96 * np.sqrt(var[:, 0]),
    color="C0",
    alpha=0.2,
)

plt.plot(xx, samples[:, :, 0].numpy().T, "C0", linewidth=0.5)
_ = plt.xlim(-0.1, 1.1)
```

## GP regression in higher dimensions

Very little changes when the input space has more than one dimension. By default, the `lengthscales` is an isotropic (scalar) parameter. It is generally recommended that you allow to tune a different lengthscale for each dimension (Automatic Relevance Determination, ARD): simply initialize `lengthscales` with an array of length $D$ corresponding to the input dimension of `X`.  See [Manipulating kernels](../advanced/kernels.ipynb) for further information.

## Further reading

  - [Stochastic Variational Inference for scalability with SVGP](../advanced/gps_for_big_data.ipynb) for cases where there are a large number of observations.
  - [Ordinal regression](../advanced/ordinal_regression.ipynb) if the data is ordinal.
  - [Multi-output models and coregionalisation](../advanced/coregionalisation.ipynb) if `Y` is multidimensional.


```
# default_exp drawings
```

```
%load_ext autoreload
%autoreload 2
```

```
from IPython.display import Image
#from IPython.core.display import HTML
from IPython.display import HTML

import numpy as np
import matplotlib.pyplot as plt
import pandas as pd

# import the Python Image processing Library
from PIL import Image as PILImage

# Makes it possible to do symbolic maths and use a control system lib
import sympy
sympy.init_printing()

# Let's also ignore some warnings here due to sympy using an old matplotlib function to render Latex equations.
import warnings
warnings.filterwarnings('ignore')

# For animations
from matplotlib import animation, rc

# equivalent to rcParams['animation.html'] = 'html5'
rc('animation', html='html5')

# Enable interactive plot
#%matplotlib notebook
```

```
%matplotlib inline
```

```
font = {'family' : 'normal',
        'weight' : 'normal',
        'size'   : 22}

## Requires latex to be installed! This is not the usual jupyter notebook install, it is the proper version
# plt.rcParams.update({
#     "text.usetex": True,
#     "font.family": "sans-serif",
#     "font.sans-serif": ["Helvetica"]})

# # for Palatino and other serif fonts use:
# plt.rcParams.update({
#     "text.usetex": True,
#     "font.family": "serif",
#     "font.serif": ["Palatino"],
# })

rc('font', **font)
```

```
plt.xkcd()  # Yes...
```

    <matplotlib.pyplot._xkcd at 0x7fea029cca90>

-----------------------

# Drawings

## Draw angles

From [ArmNavigation](https://github.com/AtsushiSakai/PythonRobotics/blob/master/ArmNavigation/two_joint_arm_to_point_control/Planar_Two_Link_IK.ipynb)

```
def transform_points(points, theta, origin):
    T = np.array([[np.cos(theta), -np.sin(theta), origin[0]],
                  [np.sin(theta), np.cos(theta), origin[1]],
                  [0, 0, 1]])
    return np.matmul(T, np.array(points))

def draw_angle(angle, offset=0, origin=[0, 0], r=0.5, n_points=100):
        x_start = r*np.cos(angle)
        x_end = r
        dx = (x_end - x_start)/(n_points-1)
        coords = [[0 for _ in range(n_points)] for _ in range(3)]
        x = x_start
        for i in range(n_points-1):
            y = np.sqrt(r**2 - x**2)
            coords[0][i] = x
            coords[1][i] = y
            coords[2][i] = 1
            x += dx
        coords[0][-1] = r
        coords[2][-1] = 1
        coords = transform_points(coords, offset, origin)
        plt.plot(coords[0], coords[1], 'k-')
```

```
draw_angle(angle=np.radians(45), offset=0, origin=[0, 0], r=0.1)

plt.axis("equal")
plt.show()
```

## Draw a complex number on the Argand diagram

```
fig = plt.figure()

def plot_complex(real, img):
    plt.arrow(0, 0, real, img, length_includes_head=True, head_width=np.linalg.norm([real, img])*0.1, head_length=np.linalg.norm([real, img])*0.1)
    plt.plot(real, img, color='red', marker='.', markersize=15)
    plt.plot([-real*10, real*10], [0, 0], color='black', linewidth=3, linestyle='--')
    plt.plot([0, 0], [-img*10, img*10], color='black', linewidth=3, linestyle='--')
    draw_angle(angle=np.arctan2(img, real), offset=0, origin=[0, 0], r=np.linalg.norm([real, img])*0.3)

    plt.xlabel('Real')
    plt.ylabel('Imag')
    plt.text(real/2-0.2, img/2+0.3, 'L', fontsize='small', fontweight='normal')
    plt.text(1, 0.3, '$\phi$', fontsize='small', fontweight='normal')

plot_complex(2, 2)

plt.axis([-1, 3, -1, 3])
plt.grid()
fig.gca().set_aspect('equal', 'box')

plt.savefig('img/37.H_jw.png', bbox_inches='tight')
```

```
fig = plt.figure()

def plot_complex(real, img, length):
    plt.arrow(0, 0, real, img, length_includes_head=True, head_width=np.linalg.norm([real, img])*0.1, head_length=np.linalg.norm([real, img])*0.1)
    plt.plot(real, img, color='red', marker='.', markersize=15)
    plt.plot([-real*10, real*10], [0, 0], color='black', linewidth=3, linestyle='--')
    plt.plot([0, 0], [-img*10, img*10], color='black', linewidth=3, linestyle='--')
    draw_angle(angle=np.arctan2(img, real), offset=0, origin=[0, 0], r=length)

    plt.xlabel('Real')
    plt.ylabel('Imag')

real, img = 2, 2
plot_complex(real, img, np.linalg.norm([real, img])*0.4)
plt.text(1.1, 0.3, '$\phi={:.0f}^o$'.format(np.degrees(np.arctan2(img, real))), fontsize='x-small', fontweight='normal')

real, img = -2, 2
plot_complex(real, img, np.linalg.norm([real, img])*0.2)
plt.text(-0.8, 1, '$\phi={:.0f}^o$'.format(np.degrees(np.arctan2(img, real))), fontsize='x-small', fontweight='normal')

real, img = 2, -2
plot_complex(real, img, np.linalg.norm([real, img])*0.2)
plt.text(0.8, -0.7, '$\phi={:.0f}^o$'.format(np.degrees(np.arctan2(img, real))), fontsize='x-small', fontweight='normal')

plt.axis([-3, 3, -3, 3])
plt.grid()
fig.gca().set_aspect('equal', 'box')

plt.savefig('img/38.H_jw_multiple_points.png', bbox_inches='tight')
```

```
fig = plt.figure()

def plot_K(real, img, length):
    plt.plot(real, img, color='red', marker='.', markersize=15)
    plt.plot([-real*10, real*10], [0, 0], color='black', linewidth=3, linestyle='--')
    plt.plot([0, 0], [-10, 10], color='black', linewidth=3, linestyle='--')
    #plt.arrow(0, 0, real, img, length_includes_head=True, head_width=np.linalg.norm([real, img])*0.1, head_length=np.linalg.norm([real, img])*0.1, color='red', linewidth=4)
    #draw_angle(angle=np.arctan2(img, real), offset=0, origin=[0, 0], r=length)

# Axis
plt.plot([-real*10, real*10], [0, 0], color='black', linewidth=3, linestyle='--')
plt.plot([0, 0], [-10, 10], color='black', linewidth=3, linestyle='--')

# K>0
real, img = 3, 0
plt.plot(real, img, color='green', marker='.', markersize=20)
plt.text(real-0.5, -0.8, 'K>0', fontsize='x-small', fontweight='normal', color='green')
plt.arrow(0, 0, real, img, length_includes_head=True, head_width=np.linalg.norm([real, img])*0.1, head_length=np.linalg.norm([real, img])*0.1, color='red', linewidth=3)

plt.plot([real, real], [0, -2.5], linestyle='--', linewidth=1, color='red') # vertical line
plt.plot([real, 0], [-2, -2], linestyle='--', linewidth=1, color='red')     # horizontal line
plt.text(1.5, -1.8, 'K', fontsize='x-small', fontweight='normal', color='red')
plt.text(0.3, +0.4, '0 phase', fontsize='x-small', fontweight='normal', color='red')

# K<0
real, img = -3, 0
plt.plot(real, img, color='blue', marker='.', markersize=20)
plt.text(real-0.5, -0.8, 'K<0', fontsize='x-small', fontweight='normal', color='blue')
plt.arrow(0, 0, real, img, length_includes_head=True, head_width=np.linalg.norm([real, img])*0.1, head_length=np.linalg.norm([real, img])*0.1, color='magenta', linewidth=3)

plt.plot([real, real], [0, -2.5], linestyle='--', linewidth=1, color='magenta')# vertical line
plt.plot([real, 0], [-2, -2], linestyle='--', linewidth=1, color='magenta') # horizontal line
plt.text(-1.8, -1.8, 'K', fontsize='x-small', fontweight='normal', color='magenta')
#draw_angle(angle=np.arctan2(img, real), offset=0, origin=[0, 0], r=0.5)
plt.text(-0.9, -0.7, '-180 phase', fontsize='x-small', fontweight='normal', color='magenta')

plt.xlabel('Real')
plt.ylabel('Imag')

plt.axis([-5, 5, -3, 3])
plt.grid()
fig.gca().set_aspect('equal', 'box')

plt.savefig('img/39.H_jw_K.png', bbox_inches='tight')
```

```
K = 1
if K:
    phi = 0
else:
    phi = -180
```

```
fig, axs = plt.subplots(1, 2, figsize=(20,5));

ws = np.logspace(-2, 2, 100) # logspace sets the exponents

gain, phase_pos, phase_neg = [], [], []

K=10
for w in ws:
    gain.append(20*np.log10(np.abs(K)))
    phase_pos.append(0)
    phase_neg.append(-180)

# Gain
axs[0].plot(ws, 20*np.log10(gain), linewidth=4, color='red')
axs[0].text( 0.01, 20*np.log10(gain[0])+10, '$20\log_{10}(|K|)$', fontsize='small', fontweight='normal', color='red')
axs[0].set_xscale('log')
axs[0].set_yscale('log')
axs[0].set_ylim((0.1, 100))
axs[0].grid('log')
axs[0].set_xlabel('Frequency (rad/s)')
axs[0].set_ylabel('Gain (dB)');

# Phase
axs[1].plot(ws, phase_pos, linewidth=3, color='green')
axs[1].plot(ws, phase_neg, linewidth=3, color='blue')

axs[1].text( 0.01, 10, '$0, K>0$', fontsize='small', fontweight='normal', color='green')
axs[1].text( 0.01, -170, '$-180, K<0$', fontsize='small', fontweight='normal', color='blue')

axs[1].set_xscale('log')
axs[1].set_ylim((-200, 180))
axs[1].grid('log')
axs[1].set_xlabel('Frequency (rad/s)')
axs[1].set_ylabel('Phase (deg)');

#fig.gca().set_aspect('equal', 'box')

plt.savefig('img/40.Bode_H_jw_K.png', bbox_inches='tight')
```

```
fig, axs = plt.subplots(1, 1, figsize=(10,5));

x = np.linspace(0, 10, 50)
plt.plot(x, np.sin(x), color='blue', linewidth=4, label="Input")
plt.plot(x, -0.8*np.sin(x), color='red', linewidth=4, label="Output K<0")
plt.plot(3.2, 0, marker='.', color='green', markersize=22)
plt.arrow(3.2, 0, 3, 0, length_includes_head=True, head_width=0.1, head_length=0.1, color='green', linewidth=3)

#plt.gca().legend(loc="center right", bbox_to_anchor=(8,0.5)) # loc=["upper left", "lower left", "center right"]
plt.legend(loc="lower right", bbox_to_anchor=(1,1))
plt.grid()

plt.savefig('img/42sinwt_shift', bbox_inches='tight')
```

```
fig = plt.figure()

def plot_complex(real, img, length):

    plt.plot([-10, 10], [0, 0], color='black', linewidth=3, linestyle='--')
    plt.plot([0, 0], [-img*10, img*10], color='black', linewidth=3, linestyle='--')

    plt.plot(real, img, color='red', marker='.', markersize=15)

    plt.arrow(0, 0, real, img, length_includes_head=True,
              head_width=np.linalg.norm([real, img])*0.1,
              head_length=np.linalg.norm([real, img])*0.1,
              color = 'red',
              linewidth=3
             )
    plt.xlabel('Real')
    plt.ylabel('Imag')

w = 0.5

real, img = 0, -1/w
plot_complex(real, img, np.linalg.norm([real, img])*0.2)
plt.text(0.3, -0.7, '$\phi={:.0f}^o$'.format(np.degrees(np.arctan2(img, real))), fontsize='x-small', fontweight='normal')
plt.text(0.3, -2, '$-1/\omega$', fontsize='x-small', fontweight='normal')

#draw_angle(angle=np.arctan2(img, real), offset=0, origin=[0, 0], r=length)

plt.axis([-3, 3, -3, 3])
plt.grid()
fig.gca().set_aspect('equal', 'box')

plt.savefig('img/43.H_jw_integrator_bode.png', bbox_inches='tight')
```

```
fig, axs = plt.subplots(2, 1, figsize=(10, 7))
ws = np.logspace(-3,2,50)

w_0 = 1

# w-axis
axs[0].loglog(ws, np.ones((len(ws),1)), color='black', linestyle='--', linewidth=2)
axs[1].plot(ws, np.zeros((len(ws),1)), color='black', linestyle='--', linewidth=2)

# plot w_0
axs[0].loglog(w_0, 1, color='black', marker='|', markersize=12)
axs[0].text(w_0, 2, '$w_0$')
axs[1].plot(w_0, 0, color='black', marker='|', markersize=12)
axs[1].text(w_0, 20, '$w_0$')

# plot points
axs[0].loglog(1e-2, 1, color='red', marker='.', markersize=15)
axs[1].plot(1e-2, 0, color='red', marker='.', markersize=15)

axs[1].set_xscale('log')

axs[0].set_ylim(0.01, 100)
axs[1].set_ylim(-190, 190)
axs[0].grid('log')
axs[1].grid('log')
axs[0].set_ylabel('Gain (dB)')
axs[1].set_ylabel('Phase (deg)')
plt.xlabel('Frequency (rad/s)')

fig.tight_layout()
plt.savefig('img/45.H_jw_real_pole_bode_1.png', bbox_inches='tight')
```

```
fig, axs = plt.subplots(2, 1, figsize=(10, 7))
ws = np.logspace(-3,2,50)

w_0 = 1

# w-axis
axs[0].plot(ws, np.zeros((len(ws),1)), color='black', linestyle='--', linewidth=2)
axs[1].plot(ws, np.zeros((len(ws),1)), color='black', linestyle='--', linewidth=2)

# plot w_0
axs[0].plot(w_0, 20*np.log10(1), color='black', marker='|', markersize=12)
axs[0].text(w_0, 1, '$w_0$')
axs[1].plot(w_0, 0, color='black', marker='|', markersize=12)
axs[1].text(w_0, 20, '$w_0$')

# plot points
axs[0].plot(1e-2, 20*np.log10(1), color='red', marker='.', markersize=15)
axs[1].plot(1e-2, 0, color='red', marker='.', markersize=15)

axs[0].set_xscale('log')
axs[1].set_xscale('log')

axs[0].set_ylim(-10, 10)
axs[1].set_ylim(-190, 190)
axs[0].grid('log')
axs[1].grid('log')
axs[0].set_ylabel('Gain (dB)')
axs[1].set_ylabel('Phase (deg)')
plt.xlabel('Frequency (rad/s)')

fig.tight_layout()
plt.savefig('img/45.H_jw_real_pole_bode_0.png', bbox_inches='tight')
```

```
fig, axs = plt.subplots(2, 1, figsize=(10, 7))
ws = np.logspace(-3,2,50)

w_0 = 1

# w-axis
axs[0].plot(ws, np.zeros((len(ws),1)), color='black', linestyle='--', linewidth=2)
axs[1].plot(ws, np.zeros((len(ws),1)), color='black', linestyle='--', linewidth=2)

# plot w_0
axs[0].plot(w_0, 20*np.log10(1), color='black', marker='|', markersize=12)
axs[0].text(w_0, 1, '$w_0$')
axs[1].plot(w_0, 0, color='black', marker='|', markersize=12)
axs[1].text(w_0, 20, '$w_0$')

# plot points
# w << w_0
axs[0].plot(1e-2, 20*np.log10(1), color='red', marker='.', markersize=15)
axs[1].plot(1e-2, 0, color='red', marker='.', markersize=15)

# w = w_0
axs[0].plot(w_0, -20*np.log10(np.sqrt(2)), color='blue', marker='.', markersize=15) # gain
axs[1].plot(w_0, -45, color='blue', marker='.', markersize=15) # phase

axs[0].set_xscale('log')
axs[1].set_xscale('log')

axs[0].set_ylim(-10, 10)
axs[1].set_ylim(-190, 190)
axs[0].grid('log')
axs[1].grid('log')
axs[0].set_ylabel('Gain (dB)')
axs[1].set_ylabel('Phase (deg)')
plt.xlabel('Frequency (rad/s)')

fig.tight_layout()
plt.savefig('img/46.H_jw_real_pole_bode_1_1.png', bbox_inches='tight')
```

```
fig, axs = plt.subplots(2, 1, figsize=(10, 7))
ws = np.logspace(-3,2,50)

w_0 = 1

# w-axis
axs[0].plot(ws, np.zeros((len(ws),1)), color='black', linestyle='--', linewidth=2)
axs[1].plot(ws, np.zeros((len(ws),1)), color='black', linestyle='--', linewidth=2)

# plot w_0
axs[0].plot(w_0, 20*np.log10(1), color='black', marker='|', markersize=12)
axs[0].text(w_0, 1, '$w_0$')
axs[1].plot(w_0, 0, color='black', marker='|', markersize=12)
axs[1].text(w_0, -10, '$w_0$')

# plot points
# w << w_0
axs[0].plot(1e-2, 20*np.log10(1), color='red', marker='.', markersize=15)
axs[1].plot(1e-2, 0, color='red', marker='.', markersize=15)

# w = w_0
axs[0].plot(w_0, -20*np.log10(np.sqrt(2)), color='blue', marker='.', markersize=15) # gain
axs[1].plot(w_0, -45, color='blue', marker='.', markersize=15) # phase

# w >> w_0
axs[0].plot([w_0, 10*w_0, 100*w_0], [-20*np.log10(1), -20*np.log10(10), -20*np.log10(100)], color='green', marker='.', markersize=15) # gain
axs[1].plot(100*w_0, -90, color='green', marker='.', markersize=15) # phase

# asymptotic plot
axs[0].plot([w_0, 10*w_0, 100*w_0], [-20*np.log10(1), -20*np.log10(10), -20*np.log10(100)], color='green', marker='.', markersize=15, linewidth=4) # gain
axs[0].plot([1e-3, w_0], [-20*np.log10(1), -20*np.log10(1)], color='green', marker='', markersize=15, linewidth=4) # gain

axs[1].plot([1e-3, w_0], [0, 0], color='green', marker='', markersize=15, linewidth=4) # phase
axs[1].plot([w_0, 100*w_0], [-90, -90], color='green', marker='', markersize=15, linewidth=4) # phase
axs[1].plot([w_0, w_0], [0, -90], color='green', marker='', markersize=15, linewidth=4) # phase

axs[0].set_xscale('log')
axs[1].set_xscale('log')

axs[0].set_ylim(-40, 10)
axs[1].set_ylim(-100, 10)
axs[1].set_yticks([0, -45, -90])
axs[0].grid('log')
axs[1].grid('log')
axs[0].set_ylabel('Gain (dB)')
axs[1].set_ylabel('Phase (deg)')
plt.xlabel('Frequency (rad/s)')

fig.tight_layout()
plt.savefig('img/47.H_jw_real_pole_bode_2.png', bbox_inches='tight')
```

```
fig, axs = plt.subplots(2, 1, figsize=(10, 7))
ws = np.logspace(-3,2,50)

w_0 = 1

# w-axis
axs[0].plot(ws, np.zeros((len(ws),1)), color='black', linestyle='--', linewidth=2)
axs[1].plot(ws, np.zeros((len(ws),1)), color='black', linestyle='--', linewidth=2)

# plot w_0
axs[0].plot(w_0, 20*np.log10(1), color='black', marker='|', markersize=12)
axs[0].text(w_0, 1, '$w_0$')
axs[1].plot(w_0, 0, color='black', marker='|', markersize=12)
axs[1].text(w_0, -10, '$w_0$')

axs[1].plot(0.1*w_0, 0, color='black', marker='|', markersize=12)
axs[1].text(0.1*w_0, -10, '$0.1 w_0$')

axs[1].plot(10*w_0, 0, color='black', marker='|', markersize=12)
axs[1].text(10*w_0, -10, '$10 w_0$')

# plot points
# w << w_0
axs[0].plot(1e-2, 20*np.log10(1), color='red', marker='.', markersize=15)
axs[1].plot(1e-2, 0, color='red', marker='.', markersize=15)

# w = w_0
axs[0].plot(w_0, -20*np.log10(np.sqrt(2)), color='blue', marker='.', markersize=15) # gain
axs[1].plot(w_0, -45, color='blue', marker='.', markersize=15) # phase

# w >> w_0
axs[0].plot([w_0, 10*w_0, 100*w_0], [-20*np.log10(1), -20*np.log10(10), -20*np.log10(100)], color='green', marker='.', markersize=15) # gain
axs[1].plot(100*w_0, -90, color='green', marker='.', markersize=15) # phase

# asymptotic plot
axs[0].plot([w_0, 10*w_0, 100*w_0], [-20*np.log10(1), -20*np.log10(10), -20*np.log10(100)], color='green', marker='.', markersize=15, linewidth=4) # gain
axs[0].plot([1e-3, w_0], [-20*np.log10(1), -20*np.log10(1)], color='green', marker='', markersize=15, linewidth=4) # gain

# axs[1].plot([1e-3, w_0], [0, 0], color='green', marker='', markersize=15, linewidth=4) # phase
# axs[1].plot([w_0, 100*w_0], [-90, -90], color='green', marker='', markersize=15, linewidth=4) # phase
# axs[1].plot([w_0, w_0], [0, -90], color='green', marker='', markersize=15, linewidth=4) # phase

# better approximation of the phase
axs[1].plot([1e-3, 0.1*w_0], [0, 0], color='magenta', marker='', markersize=15, linewidth=2) # phase
axs[1].plot([10*w_0, 100*w_0], [-90, -90], color='magenta', marker='', markersize=15, linewidth=4) # phase
axs[1].plot([0.1*w_0, 10*w_0], [0, -90], color='magenta', marker='', markersize=15, linewidth=4) # phase

axs[0].set_xscale('log')
axs[1].set_xscale('log')

axs[0].set_ylim(-40, 10)
axs[1].set_ylim(-100, 10)
axs[1].set_yticks([0, -45, -90])
axs[0].grid('log')
axs[1].grid('log')
axs[0].set_ylabel('Gain (dB)')
axs[1].set_ylabel('Phase (deg)')
plt.xlabel('Frequency (rad/s)')

fig.tight_layout()
plt.savefig('img/47.H_jw_real_pole_bode_2_v2.png', bbox_inches='tight')
```

Complex poles Bode plot

```
fig, axs = plt.subplots(2, 1, figsize=(10, 7))
ws = np.logspace(-3,2,50)

w_0 = 1

# w-axis
axs[0].plot(ws, np.zeros((len(ws),1)), color='black', linestyle='--', linewidth=2)
axs[1].plot(ws, np.zeros((len(ws),1)), color='black', linestyle='--', linewidth=2)

# plot w_0
axs[0].plot(w_0, 20*np.log10(1), color='black', marker='|', markersize=12)
axs[0].text(w_0, 1, '$w_0$')
axs[1].plot(w_0, 0, color='black', marker='|', markersize=12)
axs[1].text(w_0, -10, '$w_0$')

# axs[1].plot(0.1*w_0, 0, color='black', marker='|', markersize=12)
# axs[1].text(0.1*w_0, -10, '$0.1 w_0$')

# axs[1].plot(10*w_0, 0, color='black', marker='|', markersize=12)
# axs[1].text(10*w_0, -10, '$10 w_0$')

# plot points
# w << w_0
# axs[0].plot(1e-2, 20*np.log10(1), color='red', marker='.', markersize=15)
# axs[1].plot(1e-2, 0, color='red', marker='.', markersize=15)

# w = w_0
# axs[0].plot(w_0, -20*np.log10(np.sqrt(2)), color='blue', marker='.', markersize=15) # gain
# axs[1].plot(w_0, -45, color='blue', marker='.', markersize=15) # phase

# w >> w_0
# axs[0].plot([w_0, 10*w_0, 100*w_0], [-20*np.log10(1), -20*np.log10(10), -20*np.log10(100)], color='green', marker='.', markersize=15) # gain
# axs[1].plot(100*w_0, -90, color='green', marker='.', markersize=15) # phase

# asymptotic plot
axs[0].plot([1e-3, w_0], [-20*np.log10(1), -20*np.log10(1)], color='green', marker='', markersize=15, linewidth=4) # gain
#axs[0].plot([w_0, 10*w_0, 100*w_0], [-20*np.log10(1), -20*np.log10(10), -20*np.log10(100)], color='green', marker='.', markersize=15, linewidth=4) # gain

axs[1].plot([1e-3, w_0], [0, 0], color='green', marker='', markersize=15, linewidth=4) # phase
# axs[1].plot([w_0, 100*w_0], [-90, -90], color='green', marker='', markersize=15, linewidth=4) # phase
# axs[1].plot([w_0, w_0], [0, -90], color='green', marker='', markersize=15, linewidth=4) # phase

# better approximation of the phase
# axs[1].plot([1e-3, 0.1*w_0], [0, 0], color='magenta', marker='', markersize=15, linewidth=2) # phase
# axs[1].plot([10*w_0, 100*w_0], [-90, -90], color='magenta', marker='', markersize=15, linewidth=4) # phase
# axs[1].plot([0.1*w_0, 10*w_0], [0, -90], color='magenta', marker='', markersize=15, linewidth=4) # phase

axs[0].set_xscale('log')
axs[1].set_xscale('log')

axs[0].set_ylim(-40, 10)
axs[1].set_ylim(-100, 10)
axs[1].set_yticks([0, -45, -90])
axs[0].grid('log')
axs[1].grid('log')
axs[0].set_ylabel('Gain (dB)')
axs[1].set_ylabel('Phase (deg)')
plt.xlabel('Frequency (rad/s)')

fig.tight_layout()
plt.savefig('img/50.H_jw_complex_pole_bode_1.png', bbox_inches='tight')
```

```
fig, axs = plt.subplots(2, 1, figsize=(10, 7))
ws = np.logspace(-3,2,50)

w_0 = 1

# w-axis
axs[0].plot(ws, np.zeros((len(ws),1)), color='black', linestyle='--', linewidth=2)
axs[1].plot(ws, np.zeros((len(ws),1)), color='black', linestyle='--', linewidth=2)

# plot w_0
axs[0].plot(w_0, 20*np.log10(1), color='black', marker='|', markersize=12)
axs[0].text(w_0, 1, '$w_0$')
axs[1].plot(w_0, 0, color='black', marker='|', markersize=12)
axs[1].text(w_0, -10, '$w_0$')

# axs[1].plot(0.1*w_0, 0, color='black', marker='|', markersize=12)
# axs[1].text(0.1*w_0, -10, '$0.1 w_0$')

# axs[1].plot(10*w_0, 0, color='black', marker='|', markersize=12)
# axs[1].text(10*w_0, -10, '$10 w_0$')

# plot points
# w << w_0
# axs[0].plot(1e-2, 20*np.log10(1), color='red', marker='.', markersize=15)
# axs[1].plot(1e-2, 0, color='red', marker='.', markersize=15)

# w = w_0
# axs[0].plot(w_0, -20*np.log10(np.sqrt(2)), color='blue', marker='.', markersize=15) # gain
# axs[1].plot(w_0, -45, color='blue', marker='.', markersize=15) # phase

# w >> w_0
axs[0].plot([w_0, 10*w_0, 100*w_0], [-40*np.log10(1), -40*np.log10(10), -40*np.log10(100)], color='magenta', marker='.', markersize=15) # gain
axs[1].plot(100*w_0, -180, color='magenta', marker='.', markersize=15) # phase

# asymptotic plot
axs[0].plot([w_0, 10*w_0, 100*w_0], [-40*np.log10(1), -40*np.log10(10), -40*np.log10(100)], color='magenta', marker='.', markersize=15, linewidth=4) # gain
axs[0].plot([1e-3, w_0], [-40*np.log10(1), -40*np.log10(1)], color='green', marker='', markersize=15, linewidth=4) # gain

axs[1].plot([1e-3, w_0], [0, 0], color='green', marker='', markersize=15, linewidth=4) # phase
axs[1].plot([w_0, 100*w_0], [-180, -180], color='magenta', marker='', markersize=15, linewidth=4) # phase
#axs[1].plot([w_0, w_0], [0, -180], color='green', marker='', markersize=15, linewidth=4) # phase

# better approximation of the phase
# axs[1].plot([1e-3, 0.1*w_0], [0, 0], color='magenta', marker='', markersize=15, linewidth=2) # phase
# axs[1].plot([10*w_0, 100*w_0], [-180, -180], color='magenta', marker='', markersize=15, linewidth=4) # phase
# axs[1].plot([0.1*w_0, 10*w_0], [0, -180], color='magenta', marker='', markersize=15, linewidth=4) # phase

axs[0].set_xscale('log')
axs[1].set_xscale('log')

axs[0].set_ylim(-100, 10)
axs[1].set_ylim(-200, 10)
axs[0].set_yticks([0, -20, -40, -60, -80, -100])
axs[1].set_yticks([0, -45, -90,-135,-180])
axs[0].grid('log')
axs[1].grid('log')
axs[0].set_ylabel('Gain (dB)')
axs[1].set_ylabel('Phase (deg)')
plt.xlabel('Frequency (rad/s)')

fig.tight_layout()
plt.savefig('img/51.H_jw_complex_pole_bode_w_gg_w0_1.png', bbox_inches='tight')
```

```
fig, axs = plt.subplots(2, 1, figsize=(10, 7))
ws = np.logspace(-3,2,50)

w_0 = 1

# w-axis
axs[0].plot(ws, np.zeros((len(ws),1)), color='black', linestyle='--', linewidth=2)
axs[1].plot(ws, np.zeros((len(ws),1)), color='black', linestyle='--', linewidth=2)

# plot w_0
axs[0].plot(w_0, 20*np.log10(1), color='black', marker='|', markersize=12)
axs[0].text(w_0, 1, '$w_0$')
axs[1].plot(w_0, 0, color='black', marker='|', markersize=12)
axs[1].text(w_0, -10, '$w_0$')

# axs[1].plot(0.1*w_0, 0, color='black', marker='|', markersize=12)
# axs[1].text(0.1*w_0, -10, '$0.1 w_0$')

# axs[1].plot(10*w_0, 0, color='black', marker='|', markersize=12)
# axs[1].text(10*w_0, -10, '$10 w_0$')

# plot points
# w << w_0
# axs[0].plot(1e-2, 20*np.log10(1), color='red', marker='.', markersize=15)
# axs[1].plot(1e-2, 0, color='red', marker='.', markersize=15)

# w = w_0
#axs[0].plot(w_0, -20*np.log10(np.sqrt(2)), color='blue', marker='.', markersize=15) # gain
axs[1].plot(w_0, -90, color='blue', marker='.', markersize=15) # phase

# w >> w_0
axs[0].plot([w_0, 10*w_0, 100*w_0], [-40*np.log10(1), -40*np.log10(10), -40*np.log10(100)], color='magenta', marker='.', markersize=15) # gain
axs[1].plot(100*w_0, -180, color='magenta', marker='.', markersize=15) # phase

# asymptotic plot
axs[0].plot([w_0, 10*w_0, 100*w_0], [-40*np.log10(1), -40*np.log10(10), -40*np.log10(100)], color='magenta', marker='.', markersize=15, linewidth=4) # gain
axs[0].plot([1e-3, w_0], [-40*np.log10(1), -40*np.log10(1)], color='green', marker='', markersize=15, linewidth=4) # gain

axs[1].plot([1e-3, w_0], [0, 0], color='green', marker='', markersize=15, linewidth=4) # phase
axs[1].plot([w_0, w_0], [-0, -180], color='black', marker='', markersize=15, linewidth=4, linestyle='--') # phase
axs[1].plot([w_0, 100*w_0], [-180, -180], color='magenta', marker='', markersize=15, linewidth=4) # phase

axs[0].plot([w_0, w_0], [0, 10], color='blue', markersize=15, linewidth=4) # gain

#axs[1].plot([w_0, w_0], [0, -180], color='green', marker='', markersize=15, linewidth=4) # phase

# better approximation of the phase
# axs[1].plot([1e-3, 0.1*w_0], [0, 0], color='magenta', marker='', markersize=15, linewidth=2) # phase
# axs[1].plot([10*w_0, 100*w_0], [-180, -180], color='magenta', marker='', markersize=15, linewidth=4) # phase
# axs[1].plot([0.1*w_0, 10*w_0], [0, -180], color='magenta', marker='', markersize=15, linewidth=4) # phase

axs[0].set_xscale('log')
axs[1].set_xscale('log')

axs[0].set_ylim(-100, 20)
axs[1].set_ylim(-200, 10)
axs[0].set_yticks([0, -20, -40, -60, -80, -100])
axs[1].set_yticks([0, -45, -90,-135,-180])
axs[0].grid('log')
axs[1].grid('log')
axs[0].set_ylabel('Gain (dB)')
axs[1].set_ylabel('Phase (deg)')
plt.xlabel('Frequency (rad/s)')

fig.tight_layout()
plt.savefig('img/52.H_jw_complex_pole_bode_w_eq_w0_1.png', bbox_inches='tight')
```

# Final values

```
fig, axs = plt.subplots(1, 4, figsize=(20, 5))

t = np.linspace(0, 20, 50)

axs[0].plot(t, np.exp(-.5*t), color='red', linewidth=4)
axs[0].set_xlabel('Time (s)')
axs[0].set_title('FV = 0')
axs[0].grid()

axs[1].plot(t, t/(t+1), color='red', linewidth=4)
axs[1].set_xlabel('Time (s)')
axs[1].set_title('FV = 1')
axs[1].grid()

axs[2].plot(t, np.sin(t), color='red', linewidth=4)
axs[2].set_xlabel('Time (s)')
axs[2].set_title('FV = Undefined')
axs[2].grid()

axs[3].plot(t, np.exp(.1*t), color='red', linewidth=4)
axs[3].set_xlabel('Time (s)')
axs[3].set_title('FV = Inf')
axs[3].grid()

fig.tight_layout()
plt.savefig('img/55.final_values.png', bbox_inches='tight')
```

```
plt.xkcd()  # Yes...
```

    <matplotlib.pyplot._xkcd at 0x128e00430>

```
fig, axs = plt.subplots(1, 4, figsize=(20, 5))

t = np.linspace(0, 20, 50)

axs[0].plot(t, np.exp(-.5*t), color='red', linewidth=4)
axs[0].annotate('$\\frac{1}{(s+0.5)}$', xy=(3, 0.75),  xycoords='data',
            xytext=(0.8, 0.95), textcoords='axes fraction',
            arrowprops=dict(color='red', facecolor='red', shrink=0.05),
            horizontalalignment='right', verticalalignment='top',
            fontsize=30
            )

axs[0].plot(t, np.zeros(len(t)), color='magenta', linewidth=4)
axs[0].annotate('$1$', xy=(2, 0.02),  xycoords='data',
            xytext=(0.5, 0.55), textcoords='axes fraction',
            arrowprops=dict(color='magenta', facecolor='magenta', shrink=0.05),
            horizontalalignment='right', verticalalignment='top',
            fontsize=30
            )
axs[0].set_xlabel('Time (s)')
axs[0].set_title('Type 0 - FV = 0')
axs[0].grid()

axs[1].plot(t, np.ones(len(t)), color='red', linewidth=4)
axs[1].text(2, 1.2, '$\\frac{1}{s}$', fontsize=30)
axs[1].set_ylim(0,1.5)
axs[1].set_xlabel('Time (s)')
axs[1].set_title('Type 1 - FV = 1')
axs[1].grid()

axs[2].plot(t, np.exp(+0.1*t)-1, color='red', linewidth=4)
axs[2].text(2, 3, '$\\frac{1}{s^2}\\frac{1}{s+1}$', fontsize=30)
axs[2].set_ylim(0,5)
axs[2].set_xlabel('Time (s)')
axs[2].set_title('Type 2 - FV = $\\inf$')
axs[2].grid()

axs[3].plot(t, np.exp(.2*t)-1, color='red', linewidth=4)
axs[3].set_ylim(0,5)
axs[3].set_xlabel('Time (s)')
axs[3].set_title('Type 3 - FV = $\\inf$')
axs[3].grid()

fig.tight_layout()
plt.savefig('img/56.types_response.png', bbox_inches='tight')
```

```
fig = plt.figure()

# Axis
plt.plot([-10, 10], [0, 0], color='black', linewidth=3, linestyle='--')
plt.plot([0, 0], [-10, 10], color='black', linewidth=3, linestyle='--')

# K>0
plt.plot(0, 0, color='red', marker='x', markersize=12)
plt.plot(-1, 0, color='red', marker='x', markersize=12)
plt.text(-1.2, -0.7, '-1', fontweight='normal', color='red') #fontsize='small'

plt.xlabel('Real')
plt.ylabel('Imag')

plt.axis([-3, 3, -3, 3])
plt.grid()
fig.gca().set_aspect('equal', 'box')

plt.savefig('img/57_example_1_poles', bbox_inches='tight')
```

%%html
<style>
    @media print {
        * {
             box-sizing: border-box;
          }
        .row {
             display: flex;
         }
       /* Create two equal columns that sits next to each other */
       .column {
          flex: 50%;
          padding: 10px;

        }

        div.input {
          display: none;
          padding: 0;
        }
        div.output_prompt {
          display: none;
          padding: 0;
        }
        div.text_cell_render {
          padding: 1pt;
        }
        div#notebook p,
        div#notebook,
        div#notebook li,
        p {
          font-size: 10pt;
          line-height: 115%;
          margin: 0;
        }
        .rendered_html h1,
        .rendered_html h1:first-child {
          font-size: 10pt;
          margin: 3pt 0;
        }
       .rendered_html h2,
       .rendered_html h2:first-child {
          font-size: 10pt;
          margin: 3pt 0;
       }
       .rendered_html h3,
       .rendered_html h3:first-child {
         font-size: 10pt;
         margin: 3pt 0;
       }
       div.output_subarea {
         padding: 0;
       }
       div.input_prompt{
         display: none;
         padding: 0;
      }
}

<div class="row">
  <div class="column">
    Lorem ipsum dolor sit amet, consectetur adipiscing elit.    Maecenas quis nunc pulvinar urna faucibus tincidunt ut vestibulum ligula. Sed placerat sollicitudin erat, quis dapibus nibh tempor non.
      <br/>

Id | Syntax      | Description
--|:---------:|:-----------:
1|Header      | Something  here
2|More here   | Text

  </div>

  <div class="column">
Lorem ipsum dolor sit amet, consectetur adipiscing elit. Maecenas quis nunc pulvinar urna faucibus tincidunt ut vestibulum ligula. Sed placerat sollicitudin erat, quis dapibus nibh tempor non.
  <br/>

  $$
  \begin{align}
  {x} & = \sigma(y-x) \tag{3-1}\\
  {y} & = \rho x - y - xz \tag{3-2}\\
  {x+y+z} & = -\beta z + xy \tag{3-3}
  \end{align}
  $$

  </div>
</div>

```
# <div style="clear: both;">g
```

```
fig, axs = plt.subplots(1, 2, figsize=(15, 5))

time = np.linspace(0, 10, 50)

axs[0].plot(time, np.ones(len(time)), color='blue', linewidth=4)
axs[1].plot(time, np.sin(1*time), color='blue', linewidth=4)

axs[0].plot(time, 2*np.ones(len(time)), color='red', linewidth=4)
axs[1].plot(time, 2*np.sin(1*time), color='red', linewidth=4)

axs[0].set_ylim(0, 2.5);
axs[1].set_ylim(-2, 2.5);
axs[0].set_xlabel('INPUT')
axs[1].set_xlabel('OUTPUT')

plt.savefig('img/15_1_LTI-homogeneity.png', bbox_inches='tight')
```

```

```


```python
import numpy as np
from matplotlib import pyplot
```

```python
def f(x):
    return np.power(2., x)-1.
```

```python
x = np.linspace(0., 1., 100, dtype=np.float32)
y = f(x)
poly = np.polyfit(x, y, deg=2).astype(np.float32)
fit = np.polyval(poly, x.astype(np.float32))

pyplot.plot(x, y)
pyplot.plot(x, fit, color='r')
```

```python
poly
```

```python
np.finfo(np.float32)
```

```python
x = np.float32(0.)
print (bin(x.view(dtype=np.uint32).copy()))
exponent = ((x.view(dtype=np.uint32).copy() >> 23) & (255)) - 127
exponent
```

```python
np.log2(np.e)
```

```python
def exp_approx(x):
        # http://spfrnd.de/posts/2018-03-10-fast-exponential.html
        x = x * np.log2(np.e)
        xi = np.int32(np.floor(x))
        xf = x - xi
        y = np.power(np.float32(2.), xi)
        corr = np.float32(1.) + np.polyval(poly, xf).astype(np.float32)
        iview = corr.view(dtype=np.uint32)
        print (bin(iview[0]))
        iview &= np.uint32(~(255 << 23))
        iview |= (((xi + 127) << 23)).astype(np.uint32)
        print (bin(iview[0]))
        return corr
        #return y*corr
```

```python
bin(np.uint32(~(255 << 23)))
```

```python
logs = np.linspace(7., 10., 10000)
y = exp_approx(logs)
ref = np.exp(logs)
```

```python
rel_err = np.abs(ref - y) / ref
print (rel_err.max())
pyplot.plot(logs, rel_err)
pyplot.gca().set(yscale='log')
```

```python
pyplot.plot(logs, y)
pyplot.plot(logs, ref)
pyplot.gca().set(yscale='log')
```

```python
from path_guiding import ExpApproximation

logs = np.linspace(-30., 10., 1024, dtype=np.float32)
ref = np.exp(logs)

# Vectorized version
M = 8
y = np.zeros(len(logs), dtype=np.float32)
for i in range(0, 1024, M):
    y[i:i+M] = ExpApproximation(logs[i:i+M])

pyplot.plot(logs, np.abs(ref - y) / ref)
pyplot.gca().set(yscale='log')
pyplot.show()

pyplot.plot(logs, y)
pyplot.plot(logs, ref)
pyplot.gca().set(yscale='log')
pyplot.show()

del y

# Scalar version
M = 8
y = np.array([ ExpApproximation(l) for l in logs ], dtype=np.float32)
pyplot.plot(logs, np.abs(ref - y) / ref)
pyplot.gca().set(yscale='log')
pyplot.show()
```

Fast $\log_2$ approximation
--------------------------------------

Based on the floating point representation $x=m2^e$. Hence
\begin{equation}
\log_2 x = \log_2 m + e
\end{equation}
where $m \in [1,2)$.

```python
# Need a fit for values in [1,2]
x = np.linspace(0.9, 2.1, 100, dtype=np.float32)
y = np.log2(x)
log2_poly = np.polyfit(x, y, deg=3).astype(np.float32)
fit = np.polyval(log2_poly, x.astype(np.float32))

pyplot.plot(x, y)
pyplot.plot(x, fit, color='r')
```

```python
def log2_taylor(x):
    # Expansion around x=1
    x = x-np.float32(1)
    y = x*(np.float32(1.) + x*(np.float32(1./3.)*x - np.float32(1./2.)))
    y /= np.float32(np.log(2.))
    return y

def log2_approx(x):
    iview = x.view(dtype=np.uint32)
    #print ('bin view = {}'.format(bin(iview)))
    exponent = iview & np.uint32(255 << 23)
    exponent = (exponent >> 23) - 127
    mantisse = (iview & np.uint32(~(255 << 23)) | np.uint32((127 << 23)))
    mantisse = mantisse.view(dtype=np.float32)
    # The mantisse always lies between 1 and 2
    # Close to x=1, the general log2_poly-based approximation diverges from the true value.
    # So these cases are treated separately.
    if exponent == -1 and mantisse>1.6:
        # The argument lies close to 1 due to division by 2.
        return log2_taylor(mantisse/np.float32(2))
    elif exponent == 0 and mantisse<1.2:
        return log2_taylor(mantisse)
    else:
        return np.polyval(log2_poly, mantisse) + exponent
```

```python
def f(x):
    return log2_approx(np.float32(x))
```

```python
x = 2.**(-0.0000001)
y = f(x)
yref = np.log2(np.float32(x))
print(x, y, yref, (y-yref)/yref)
```

```python
np.log2(1.9999998807907104) - 1.
```

```python
np.log2(1.9999998807907104 * 0.5)
```

```python
logs = np.linspace(-2., 2., 10000, dtype=np.float32)
x = np.power(np.float32(2.),logs)

yref = np.log2(x)
y = np.array([f(x_) for x_ in x ])

rel_err = np.abs(yref - y) / np.abs(yref)
pyplot.plot(logs, rel_err)
pyplot.gca().set(yscale='log')
```

```python

```


### Regularization

#### Topics

- Introduction to regularized regression
- Ridge regression with one predictor on a grid
- Ridge regression with polynomial features on a grid
- Cross-validation
- Refitting on full training set

## Part 0: Learning Goals
In this lab we continue where we left off in Lab 4, with regularized regression.  By the end of this lab, you will be able to:

- Implement ridge and LASSO regression using `sklearn`.
- Interpret the results of ridge and LASSO regression, and compare to the results from simple and multiple linear regression.

*This lab maps on to lectures 7 and 8 and homework 4.*

```python
%matplotlib inline
import numpy as np
import scipy as sp
import matplotlib as mpl
import matplotlib.cm as cm
import matplotlib.pyplot as plt
import pandas as pd
pd.set_option('display.width', 500)
pd.set_option('display.max_columns', 100)
pd.set_option('display.notebook_repr_html', True)
import seaborn.apionly as sns
sns.set_style("whitegrid")
sns.set_context("poster")
```

## Part 1:  Introduction to regularized regression

Recall from lecture the main idea of regularization.  In the ordinary least squares problem we minimize the loss function

\begin{equation}
L(\mathbf{\beta}) = \frac{1}{n} \sum_{i = 1}^n |y_i - \mathbf{\beta}^T \mathbf{x}_i|^2,
\end{equation}

to determine regression coefficients $\mathbf{\beta}$.  Here $y_i$ is the response variable for observation $i$, and $\mathbf{x}_i$ is a vector from the predictor matrix  corresponding to observation $i$.

The general idea behind regularization is to penalize the loss function to account for possibly very large values of the coefficients $\mathbf \beta$.  The  aforementioned optimization problem is then adjusted accordingly.  Instead of minimizing $L(\mathbf{\beta})$, we minimize the regularized loss function

\begin{equation}
L_{\mathrm{reg}}(\mathbf{\beta}) = L(\mathbf{\beta}) + \lambda R(\mathbf{\beta}),
\end{equation}

where $R(\mathbf{\beta})$ is a penalty function and $\lambda$ is a scalar that weighs the relative importance of this penalty.  In this lab we will explore two regularized regression models, ridge and LASSO.  In ridge regression, the penalty function is the sum of the squares of the parameters, giving the regularized loss function

\begin{equation}
L_{\mathrm{Ridge}}(\mathbf{\beta}) = \frac{1}{n} \sum_{i = 1}^n |y_i - \mathbf{\beta}^T \mathbf{x}_i|^2 + \lambda \sum_{j = 1}^d \beta_j^2.
\end{equation}

In LASSO regression the penalty function is the sum of the magnitudes of the parameters, leading to

\begin{equation}
L_{\mathrm{LASSO}}(\mathbf{\beta}) = \frac{1}{n} \sum_{i = 1}^n |y_i - \mathbf{\beta}^T \mathbf{x}_i|^2 + \lambda \sum_{j = 1}^d |\beta_j|.
\end{equation}

We will show how these optimization problems can be solved with `sklearn` to determine the model parameters $\mathbf \beta$.  We will also show how to choose $\lambda$ appropriately via cross-validation.

Let's continue working with our data from last time. We load and split the data as in Lab 4.

```python
df=pd.read_csv("data/noisypopulation.csv")
df.head()
```

Here `x` and `y` are the predictor and measured response variables, and `f` is the true response.

```python
f = df.f.values
x = df.x.values
y = df.y.values
df.shape
```

```python
indexes=np.sort(np.random.choice(x.shape[0], size=60, replace=False))
samplex = x[indexes]
samplef = f[indexes]
sampley = y[indexes]
sample_df=pd.DataFrame(dict(x=x[indexes],f=f[indexes],y=y[indexes]))
```

We split the sample data into training and testing sets.

```python
from sklearn.model_selection import train_test_split
datasize=sample_df.shape[0]
#split dataset using the index, as we have x,f, and y that we want to split.
itrain, itest = train_test_split(np.arange(60),train_size=0.8)
xtrain= sample_df.x[itrain].values
ftrain = sample_df.f[itrain].values
ytrain = sample_df.y[itrain].values
xtest= sample_df.x[itest].values
ftest = sample_df.f[itest].values
ytest = sample_df.y[itest].values
```

## Part 2: Ridge regression with one predictor on a grid

To begin, we'll use `sklearn` to do simple linear regression on the sampled training data.  We'll then do ridge regression with the same data, setting the penalty parameter $\lambda$ to zero.  Setting $\lambda = 0$ reduces the ridge problem to the simple ordinary least squares problem, so we expect the results of these models to be identical.

```python
from sklearn.linear_model import LinearRegression
```

```python
#build the the ordinary least squares model
simp_reg = LinearRegression()
#fit the model to training data
simp_reg.fit(xtrain.reshape(len(xtrain),1), ytrain)
#save the beta coefficients
beta0_sreg = simp_reg.intercept_
beta1_sreg = simp_reg.coef_[0]
#make predictions everywhere
ypredict = lambda x : beta0_sreg + beta1_sreg*x
print("(beta0, beta1) = (%f, %f)" %(beta0_sreg, beta1_sreg))
```

We will use the above $\beta$ coefficients as a benchmark for comparision to ridge and LASSO methods.  Let's see that we get the same coefficients with ridge regression.

```python
from sklearn.linear_model import Ridge
```

For reference, [here](http://scikit-learn.org/stable/modules/generated/sklearn.linear_model.Ridge.html) is the ridge regression documentation.  Notice that the weight $\lambda$ is referred to as `alpha` in the documentation.

The snippet of code below implements the ridge regression with $\lambda = 0$.

```python
#build the ridge regression model with specified lambda, ie, alpha
ridge_reg = Ridge(alpha = 0)
#fit the model to training data
ridge_reg.fit(xtrain.reshape(-1,1), ytrain)  #xtrain.reshape(-1,1) and xtrain.reshape(len(xtrain),1) are equivalent
#save the beta coefficients
beta0_ridge = ridge_reg.intercept_
beta1_ridge = ridge_reg.coef_[0]
#make predictions everywhere
ypredict_ridge = ridge_reg.predict(x.reshape(-1,1))
print("(beta0, beta1) = (%f, %f)" %(beta0_ridge, beta1_ridge))
```

The beta coefficients for linear and ridge regressions coincide for $\lambda = 0$, as expected. We plot the data and fits.

```python
colors = sns.color_palette()
```

```python
#plot in-sample training data
plt.plot(xtrain, ytrain, 's', alpha=0.3, ms=10, label="in-sample y (observed)")
#plot population data
plt.plot(x, y, '.', alpha=0.8, label="population y");
plt.plot(x, f, color = colors[1], label="God function")
#plot simple linear regression fit
plt.plot(x, ypredict(x), alpha=0.5, label="OLS")
#plot ridge regression fit
plt.plot(x, ypredict_ridge, 'k.', lw = 1, alpha=0.3, label="ridge")
plt.xlabel('$x$')
plt.ylabel('$y$')
plt.legend(loc=4);
```

> **EXERCISE:** Play around with the values of  $\lambda$ in the ridge regression code.  Increase $\lambda$ from 0 to .01, from 0.01 to 1, from 1 to 5.  What do you observe?  What happens as $\lambda$ goes to $\infty$?

> **YOUR DISCUSSION HERE:**

## Part 3: Ridge regression with polynomial features on a grid

Now we'll make a more complex model by adding polynomial features.  Instead of building the linear model $y = \beta_0 + \beta_1 x$, we build a polynomial model $y = \beta_0 + \beta_1 x + \beta_2 x^2 + \ldots \beta_d x^d$ for some $d$ to be determined (see Lab 4 for details on choosing hyper-parameter $d$).  This regression will be linear though, since we'll be treating  $x^2, \ldots, x^d$ themselves as predictors in the linear model.  We did this in Lab 4 but it's worth a review.

We map $x$ to $1, x, x^2, \ldots, x^d$, and then build a linear regression model on this linear function of polynomial features.  To do this, we use `sklearn` to build what is known as the *Vandermonde* matrix, the generalizaiton of the predictor matrix $X$ discussed in Lab 3. For example, if we have three observations

\begin{equation*}\begin{pmatrix}
x_1 \\
x_2 \\
x_3\\
\end{pmatrix}, \end{equation*}

and we want  polynomial features up to and including degree 4, we build the predictor matrix

\begin{equation*}\begin{pmatrix}
x_1 \\
x_2 \\
x_3 \\
\end{pmatrix} \rightarrow X = \begin{bmatrix}
x_1^0 & x_1^1 & x_1^2 & x_1^3 & x_1^4\\
x_2^0 & x_2^1 & x_2^2 & x_2^3 & x_2^4\\
x_3^0 & x_3^1 & x_3^2 & x_3^3 & x_3^4\\
\end{bmatrix} =
\begin{bmatrix}
1& x_1^1 & x_1^2 & x_1^3 & x_1^4\\
1 & x_2^1 & x_2^2 & x_2^3 & x_2^4\\
1 & x_3^1 & x_3^2 & x_3^3 & x_3^4\\
\end{bmatrix}.
\end{equation*}

```python
from sklearn.preprocessing import PolynomialFeatures
```

> **EXERCISE: ** Before we continue working with the data, make a toy vector called `toy`, where

>\begin{equation}
\mathrm{toy} = \begin{pmatrix}
0 \\
2 \\
5 \\
\end{pmatrix}
.
\end{equation}
> Build the feature matrix up to (and including) degree 4. Confirm that the entries in the matrix are what you'd expect based on the above discussion.

```python
# your code here

```

We now continue working with our data.  We write a function to make polynomial features of given degrees as we did in Lab 4, and we store the features in a dictionary.

```python
def make_features(train_set, test_set, degrees):
    train_dict = {}
    test_dict = {}
    for d in degrees:
        traintestdict={}
        train_dict[d] = PolynomialFeatures(d).fit_transform(train_set.reshape(-1,1))
        test_dict[d] = PolynomialFeatures(d).fit_transform(test_set.reshape(-1,1))
    return train_dict, test_dict
```

> **EXERCISE: ** Fill in the code below to perform ridge regression on the training data for the given set of $\lambda$.  Then predict on the grid and store the results in  `ypredict_ridge`.

```python
d = 20
rows = 7
cols = 2
lambdas = [0., 1e-6, 1e-3, 1e-2, 1e-1, 1, 10]
grid_to_predict = np.arange(0, 1, .01)
train_dict, grid_dict = make_features(xtrain, grid_to_predict, range(0,d + 1))

fig, axs = plt.subplots(rows, cols, figsize=(12, 24))
axs = axs.ravel()
Xtrain = train_dict[d]

for i, lam in enumerate(lambdas):
    #your code here

    #code provided from here on
    left = 2*i
    right = 2*i + 1
    axs[left].plot(xtrain, ytrain, 's', alpha=0.3, ms=10, label="in-sample y (observed)")
    axs[left].plot(x, y, '.', alpha=0.8, label="population y")
    axs[left].plot(grid_to_predict, ypredict_ridge, 'k-', label="lambda =  %s" % str(lam))
    axs[left].set_ylabel('$y$')
    axs[left].set_ylim((0, 1))
    axs[left].set_xlim((0, 1))
    axs[left].legend(loc=2)
    coef = ridge_reg.coef_.ravel()
    axs[right].semilogy(np.abs(coef), marker='o', label="lambda = %s" % str(lam))
    axs[right].set_ylim((1e-1, 1e15))
    axs[right].yaxis.set_label_position("right")
    axs[right].set_ylabel('abs(coefficient)')
    axs[right].legend(loc='upper left')
axs[2*(rows-1)].set_xlabel("x")
axs[2*(rows-1) + 1].set_xlabel("coefficients");

```

As you can see, as we increase $\lambda$ from 0 to 1, we start out overfitting, then doing well, and then our fits develop a mind of their own irrespective of data, as the penalty term dominates.

> **EXERCISE:** What would you expect if you compared a performance metric between these models on a grid?

> **YOUR DISCUSSION HERE:**

## Part 4: Cross-validation

Let's use cross-validation to determine the critical value of $\lambda$, which we'll refer to as $\lambda^*$. To do this we use the concept of a *meta-estimator* from scikit-learn.  As the API paper from Lab 4 explains:

>In scikit-learn, model selection is supported in two distinct meta-estimators, GridSearchCV and RandomizedSearchCV. They take as input an estimator (basic or composite), whose hyper-parameters must be optimized, and a set of hyperparameter settings to search through.

The concept of a meta-estimator allows us to wrap, for example, cross-validation, or methods that build and combine simpler models or schemes. For example:

    est = Ridge()
    parameters = {"alpha": [1e-8, 1e-6, 1e-5, 5e-5, 1e-4, 5e-4, 1e-3, 1e-2, 1e-1, 1.0]}
    gridclassifier=GridSearchCV(est, param_grid=parameters, cv=4, scoring="neg_mean_squared_error")

The `GridSearchCV` replaces the manual iteration over the folds using `KFolds` and the averaging we did in Lab 4, doing it all for us. It takes a hyper-parameter grid in the shape of a dictionary as input, and sets $\lambda$ to the values you want to try, one by one. It then trains the model using cross-validation, and gets the error for each value of the hyper-parameter $\lambda$. Finally it compares the errors for the different $\lambda$'s, and picks the best choice model.

```python
from sklearn.model_selection import GridSearchCV
def cv_optimize_ridge(x, y, list_of_lambdas, n_folds=4):
    est = Ridge()
    parameters = {'alpha': list_of_lambdas}
    #the scoring parameter below is the default one in ridge, but you can use a different one
    #in the cross-validation phase if you want.
    gs = GridSearchCV(est, param_grid=parameters, cv=n_folds, scoring="neg_mean_squared_error")
    gs.fit(x, y)
    return gs
```

> **EXERCISE:** Use the function above to fit the model on the training set with 4-fold cross validation.  Save the fit as the variable `fitmodel`.

```python
# your code here
```

```python
fitmodel.best_estimator_, fitmodel.best_params_, fitmodel.best_score_
```

We also output the mean cross-validation error at different $\lambda$ (with a negative sign, as scikit-learn likes to maximize negative error which is equivalent to minimizing error).

```python
fitmodel.cv_results_
```

```python
fit_lambdas = [d['alpha'] for d in fitmodel.cv_results_['params']]
fit_scores = fitmodel.cv_results_['mean_test_score']
```

> **EXERCISE:** Plot log10-log10 plot of `-fit_scores` versus `fit_lambdas`.

```python
#your code here

```

## Part 5: Refitting on full training set

We now refit the estimator on the training set, and calculate and plot the test set error and the polynomial coefficients. Notice how many of these coefficients have been pushed to lower values or 0.

> **EXERCISE:** Assign to variable `est` the classifier obtained by fitting the entire training set using the best $\lambda$ found above.  Assign the predictions to the variable `ypredict_ridge_best`.

```python
# your code here

```

```python
#code provided from here on
fig, axs = plt.subplots(1, 2, figsize=(12, 4))
left = 0
right = 1
axs[left].plot(xtrain, ytrain, 's', alpha=0.3, ms=10, label="in-sample y (observed)")
axs[left].plot(x, y, '.', alpha=0.8, label="population y")
axs[left].plot(grid_to_predict, ypredict_ridge_best, 'k-', label="lambda =  %s" % str(lambdawechoose))
axs[left].set_ylabel('$y$')
axs[left].set_ylim((0, 1))
axs[left].set_xlim((0, 1))
axs[left].legend(loc=2)
coef = est.coef_.ravel()
axs[right].semilogy(np.abs(coef), marker='o', label="lambda = %s" % str(lambdawechoose))
axs[right].set_ylim((1e-1, 1e15))
axs[right].yaxis.set_label_position("right")
axs[right].set_ylabel('abs(coefficient)')
axs[right].legend(loc='upper left')
axs[left].set_xlabel("x")
axs[right].set_xlabel("coefficients");
```

# END OF LAB

## Part 6: Feature selection with LASSO regression
Below is a completely worked example of feature selection with LASSO, which will be helpful for homework 4.  For reference [here](http://scikit-learn.org/stable/modules/generated/sklearn.linear_model.Lasso.html) is the documentation for LASSO.

```python
from sklearn.linear_model import Lasso
#function to do lasso with cross validation
def cv_optimize_lasso(X, y, list_of_lambdas, n_folds=4):
    #build the lasso model
    clf = Lasso()
    parameters = {"alpha": list_of_lambdas}
    #the scoring parameter below is the default one in ridge, but you can use a
    #different one in the cross-validation phase if desired.
    gs = GridSearchCV(clf, param_grid=parameters, cv=n_folds, scoring="neg_mean_squared_error")
    gs.fit(X, y)
    return gs
```

```python
#List of Lambda (lol!) values
lol = [1e-8,1e-6, 1e-5, 5e-5, 1e-4, 5e-4, 1e-3, 1e-2, 1e-1, 1.0, 10.0]
#fit lasso model to training data with cross-validation
fitmodel_lasso = cv_optimize_lasso(Xtrain, ytrain, lol, n_folds=4)
```

```python
#choose the optimal lambda
lambdawechoose_lasso = fitmodel_lasso.best_params_['alpha']
#estimate with this optimal lambda
est_lasso = Lasso(alpha=lambdawechoose_lasso).fit(Xtrain,ytrain)
est_lasso
```

```python
#function that pulls out the important features
def nonzero_lasso(est, lcols):
    featuremask=(est.coef_ !=0.0)
    return pd.DataFrame(dict(feature=lcols, coef=est.coef_,
        abscoef=np.abs(est.coef_)))[featuremask].sort_values('abscoef',
                                                    ascending=False)
```

```python
#x^1, x^2, x^6, x^20 are the important features
lasso_importances=nonzero_lasso(est_lasso, list(range(d+1)))
lasso_importances.set_index("feature", inplace=True)
lasso_importances
```

```python
#function that pulls out the trivial features
def zero_lasso(est, lcols):
    featuremask=(est.coef_ ==0.0)
    return pd.DataFrame(dict(feature=lcols, coef=est.coef_,
        abscoef=np.abs(est.coef_)))[featuremask].sort_values('abscoef',
                                                    ascending=False)
```

```python
#calculate and print the trivial features.
lasso_zeros=zero_lasso(est_lasso, list(range(d+1)))
lasso_zeros.set_index("feature", inplace=True)
lasso_zeros
```

```python
#17 of the 21 features are trivial, 4 are important
len(lasso_zeros), len(lasso_importances)
```

```python

```


# Biogeme Basics: Probit Model

```python
import pandas  as pd
import numpy as np
import biogeme.database as db
import biogeme.biogeme as bio
import biogeme.models as models
import biogeme.optimization as opt
from biogeme.expressions import Beta, DefineVariable, bioNormalCdf, log, Elem
import seaborn as sns
import matplotlib.pyplot as plt
```

**Import Swissmetro data**

```python
pandas = pd.read_csv("../../Data/6-Discrete Choice Models/swissmetro.dat",sep='\t')
database = db.Database("data/swissmetro", pandas)
```

**Use collumn names as variables**

```python
globals().update(database.variables)
```

**Exclude some unwanted entries**

Rmove observations where Swissmetro was chosen (CHOICE == 2) and also remove observations where one of the two alternatives is not available

```python
CAR_AV_SP =  DefineVariable('CAR_AV_SP',CAR_AV * (SP != 0),database)
TRAIN_AV_SP =  DefineVariable('TRAIN_AV_SP',TRAIN_AV * (SP != 0),database)
exclude = (TRAIN_AV_SP == 0) + (CAR_AV_SP == 0) + ( CHOICE == 2 ) + (( PURPOSE != 1 ) * (  PURPOSE   !=  3  ) + ( CHOICE == 0 )) > 0
database.remove(exclude)
```

**Define some dummy variables**

```python
SM_COST = SM_CO * ( GA == 0 )
TRAIN_COST = TRAIN_CO * ( GA == 0 )
```

**Rescale some data**

```python
TRAIN_TT_SCALED   = DefineVariable('TRAIN_TT_SCALED',   TRAIN_TT / 100.0, database)
TRAIN_COST_SCALED = DefineVariable('TRAIN_COST_SCALED', TRAIN_COST / 100, database)
SM_TT_SCALED      = DefineVariable('SM_TT_SCALED',      SM_TT / 100.0   , database)
SM_COST_SCALED    = DefineVariable('SM_COST_SCALED',    SM_COST / 100   , database)
CAR_TT_SCALED     = DefineVariable('CAR_TT_SCALED',     CAR_TT / 100    , database)
CAR_CO_SCALED     = DefineVariable('CAR_CO_SCALED',     CAR_CO / 100    , database)
```

**Create parameters to be estimated**

`Beta`
1. name of parameter
2. default value for the parameter
3. lower bound
4. upper bound
5. flag indicating if parameter is to be estimated

```python
ASC_CAR = Beta('ASC_CAR',0,None,None,0)
ASC_TRAIN = Beta('ASC_TRAIN',0,None,None,1)
B_TIME = Beta('B_TIME',0,None,None,0)
B_COST = Beta('B_COST',0,None,None,0)
```

**Define the utility functions**

\begin{align}
V_1 & = \beta_{time}X_{Train_{TT}} + \beta_{cost}X_{Train_{cost}}\\
V_3 & = \beta_{Car} + \beta_{time}X_{Car_{TT}} + \beta_{cost}X_{Car_{cost}}\\
\end{align}

```python
V1 = B_TIME * TRAIN_TT_SCALED + \
     B_COST * TRAIN_COST_SCALED
V3 = ASC_CAR + \
     B_TIME * CAR_TT_SCALED + \
     B_COST * CAR_CO_SCALED
```

**Associate utility functions with alternatives and associate availability of alternatives**

Create a python dictionary with all utility functions

Create a python dictionary with availability of choices

```python
P = {1: bioNormalCdf(V1-V3),
     3: bioNormalCdf(V3-V1)}
```

**Define the model**

```python
logprob = log(Elem(P, CHOICE))
```

**Define the Biogeme object**

* Give the database with all variables
* Give the log likelihood model

```python
biogeme  = bio.BIOGEME(database, logprob)

biogeme.modelName = "swissmetro_probit_basic"
```

**Estimate the model**

1. A `.html` can be generated with a report of the results and can be opened with a browser
2. A `.pickle` file can also be generaetd with a snapshot with the results. This file can then be used in other scripts

```python
biogeme.generateHtml = True
biogeme.generatePickle = False

results = biogeme.estimate()

print(f"HTML file:    {results.data.htmlFileName}")
print(f"Pickle file:  {results.data.pickleFileName }")
```

    HTML file:    swissmetro_probit_basic.html
    Pickle file:  None

**Print results**

```python
betas = results.getBetaValues()
for k,v in betas.items():
    print(f"{k:10}=\t{v:.3g}")
```

    ASC_CAR   =	0.691
    B_COST    =	-0.812
    B_TIME    =	-0.297

**Get the general statistics**

```python
gs = results.getGeneralStatistics()

for k,v in gs.items():
    print("{}= {}".format(k.ljust(45),v[0]))
```

    Number of estimated parameters               = 3
    Sample size                                  = 2232
    Excluded observations                        = 8496
    Init log likelihood                          = -1547.104507009801
    Final log likelihood                         = -986.1887862987081
    Likelihood ratio test for the init. model    = 1121.8314414221857
    Rho-square for the init. model               = 0.3625583909617164
    Rho-square-bar for the init. model           = 0.3606192847239623
    Akaike Information Criterion                 = 1978.3775725974162
    Bayesian Information Criterion               = 1995.50953256792
    Final gradient norm                          = 0.007383834715391203
    Nbr of threads                               = 8


```python
import numpy as np
from sympy import symbols, S, simplify, factor

x0, y0, z0, lamb = symbols('x0 y0 z0 lamb')

a = S(1)/2
b = -(z0 + 1)
c = (S(1)/2 + 2*z0)
d = x0**2 + y0**2 - z0

discr = simplify(18*a*b*c*d - 4*b**3*d + b**2*c**2 - 4*a*c**3 - 27*a**2*d**2)
```

```python
discr
```

$\displaystyle 4 \left(z_{0} + 1\right)^{3} \left(x_{0}^{2} + y_{0}^{2} - z_{0}\right) + \frac{\left(z_{0} + 1\right)^{2} \left(4 z_{0} + 1\right)^{2}}{4} - \frac{9 \left(z_{0} + 1\right) \left(4 z_{0} + 1\right) \left(x_{0}^{2} + y_{0}^{2} - z_{0}\right)}{2} - \frac{\left(4 z_{0} + 1\right)^{3}}{4} - \frac{27 \left(x_{0}^{2} + y_{0}^{2} - z_{0}\right)^{2}}{4}$

```python
factor(discr)
```

$\displaystyle - \frac{\left(x_{0}^{2} + y_{0}^{2}\right) \left(27 x_{0}^{2} + 27 y_{0}^{2} - 16 z_{0}^{3} + 24 z_{0}^{2} - 12 z_{0} + 2\right)}{4}$

```python
discr.subs([(x0, 1), (y0, 2), (z0, 10)])
```

$\displaystyle \frac{67915}{4}$

```python
x0s = 0
y0s = 0
z0s = 0
subs_list = [(x0, x0s), (y0, y0s), (z0, z0s)]
poly = [float(expr.subs(subs_list)) for expr in (a,b,c,d)]
lamb = np.roots(poly)
x = x0s/(1 - lamb)
y = y0s/(1 - lamb)
z = z0s - lamb/2
f = (x - x0s)**2 + (y - y0s)**2 + (z - z0s)**2
g = x**2 + y**2 - z
```

    /Users/daneaustin/miniconda3/envs/fathom/lib/python3.7/site-packages/ipykernel_launcher.py:7: RuntimeWarning: invalid value encountered in true_divide
      import sys
    /Users/daneaustin/miniconda3/envs/fathom/lib/python3.7/site-packages/ipykernel_launcher.py:8: RuntimeWarning: invalid value encountered in true_divide

```python
lamb, f, float(discr.subs(subs_list))
```

    (array([1., 1., 0.]), array([nan, nan,  0.]), 0.0)

notes/Distance from point to paraboloid - roots study.ipynb