# Profils de succion et drainage latéral dans les barrières capillaires

Pour exécuter une cellule, `Ctrl + Enter`. Pour exécuter une cellule et passer à la suivante, `Maj. + Enter`. Pour exécuter toute la feuille, dans le menu `Cell`, sélectionner `Run All`.

## Charger les librairies

```python
%pylab inline
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib.ticker as plticker
from scipy.integrate import quad
from scipy.interpolate import interp1d
pylab.rcParams['figure.figsize'] = (8.0, 6.0)
```

    Populating the interactive namespace from numpy and matplotlib

## Modèles hydrauliques

### Courbe de rétention d'eau, van Genuchten (1980):
\begin{align}
\theta(\psi)  = \theta_{r} + (\theta_{s} - \theta_{r})  (1+(a_{VG}  \psi)^{n_{VG}})^{-m_{VG}} \\
\end{align}

### Fonction de conductivité hydraulique, van Genuchten et al. (1991), basé sur van Genuchten (1980) et Mualem (1976):
\begin{align}
k(\psi) = k_{sat}  \frac {(1-((a_{VG}  \psi)^{n_{VG}m_{VG}})  (1+(a_{VG} \psi)^{n_{VG}})^{-m_{VG}}))^2} { (1+(a_{VG} \psi)^{n_{VG}})^{m_{VG}l_{VG}}}
\end{align}

```python
class VanGenuchten(object):
    """
    Returns the water retention curve and hydraulic conductivity function
    according to van Genuchten (1980)'s functions
    """
    def __init__(self, thR, thS, aVG, nVG, mVG, ksat, psi, lVG=0.5):
        self.aVG = aVG
        self.nVG = nVG
        self.mVG = mVG
        self.lVG = lVG
        self.ksat = ksat
        self.thR = thR
        self.thS = thS
        self.npoints = len(psi)
        self.psi = psi

    def wrc(self):
        self.th = self.thR + (self.thS - self.thR) * (1+(self.aVG * self.psi) ** self.nVG) ** (-self.mVG)
        return([self.psi, self.th])

    def hcf(self):
        self.k = self.ksat*((1-((self.aVG*self.psi)**(self.nVG*self.mVG))* \
        ((1+((self.aVG*self.psi)**self.nVG))**(-self.mVG)))**2) / \
        ((1+((self.aVG*self.psi)**self.nVG))**(self.mVG*self.lVG))
        return([self.psi, self.k])
```

## Profile de succion dans une colonne de matériau poreux soumis à un débit unitaire

Une démonstration similaire a d'abord été publiée parKisch (1959).

\begin{align}
q = k(\psi) \frac{dh}{dz} \\
h = z + p = z - \psi \\
q = k(\psi) \frac{dz - d\psi}{dz} \\
q = k(\psi) (1 - \frac{d\psi}{dz}) \\
\frac {q}{k(\psi)} = 1 - \frac{d\psi}{dz} \\
\frac{d\psi}{dz} = 1-\frac {q}{k(\psi)} \\
dz = \frac{d\psi}{1-\frac {q}{k(\psi)}} \\
z(\psi) = \int_{\psi_{min}}^{\psi} \frac{1}{1-\frac {q}{k(\psi)}}d\psi
\end{align}

L'intégrale peut être approximée par:
\begin{align}
z(\psi) = \sum_{i=1}^{n} \frac{\Delta\psi}{1-\frac {q}{k_n(\psi)}}
\end{align}

```python
class VerticalFlow(object):
    """
    This class uses the vertical flow model presented by Kisch (1959).
    """
    def __init__(self, model, q, psiMin = 1e-3, zMin = 0, perc = 0.999):
        self.model = model
        self.q = q
        self.psiMin = psiMin
        self.zMin = zMin
        self.perc = perc

    def kisch(self):
        # suction vector
        self.psiModel = self.model.hcf()[0]
        self.kModel = self.model.hcf()[1]

        #Find the maximal succion that can be found in a column of a por. mat. described by model under flow q
        interpFuncModel = interp1d(np.log10(self.kModel[::-1]), np.log10(self.psiModel[::-1]))

        # succion varies from the minimum succion in the model to the maximal succion value psiMax by deltaPsi
        self.psiMax = 10**interpFuncModel(np.log10(self.q))

        # recompute the model between psiMin and psiMax
        self.psiNewMod = np.logspace(start = np.log10(self.psiMin), stop = np.log10(self.psiMax),
                                     num = self.model.npoints, endpoint = True)
        newModel = VanGenuchten(thR = self.model.thR, thS = self.model.thS,
                                aVG = self.model.aVG, nVG = self.model.nVG, mVG = self.model.mVG,
                                lVG = self.model.lVG, ksat = self.model.ksat, psi = self.psiNewMod)

        self.psiKisch = newModel.hcf()[0] # suction vector
        self.kKisch = newModel.hcf()[1] # hydraulic conductivity vector

        # delta psi
        self.psiKischB = np.append(self.psiMin, self.psiKisch)
        self.deltaPsiKisch = []
        for i in range(1, len(self.psiKischB)):
            d = self.psiKischB[i] - self.psiKischB[i-1]
            self.deltaPsiKisch.append(d)
        self.deltaPsiKisch = np.array(self.deltaPsiKisch)

        # kisch
        zPieces = self.deltaPsiKisch / (1 - self.q/self.kKisch) # pieces to sum
        self.z = []
        for i in range(0, len(self.deltaPsiKisch)):
            self.z.append(np.sum(zPieces[0:i]))
        self.z = np.array(self.z) + self.zMin

        return([self.psiKisch, self.z, self.psiMax])
```

## Matériaux

Exécuter la cellule qui correpond à vos matériaux.

```python
# CBL parameters
cblthR = 0.017
cblthS = 0.37
cblaVG = 3.07 * 9.807 # 9.807 pour convertir en en m-1
cblnVG = 3.33
cblmVG = 1 - 1/cblnVG
cbllVG = 0.5
cblksat = 2.3e-3 # m/s

# MRL parameters
mrlthR = 0.1 # 0.03 #
mrlthS = 0.441794820254646 # 0.4 #
mrlaVG = 0.16162092033419 * 9.807 #0.347 * 9.807 # 9.807 pour convertir en en m-1
mrlnVG = 1.21995996561548 # 2.29 #
mrlmVG = 1 - 1/mrlnVG
mrllVG = 0.5
mrlksat = 1.9e-7#8e-5 #  m/s
```

## Modèle d'infiltration vertical

### Créer des instances pour les deux matériaux

```python
npoints = 500
psiCBL = np.logspace(start = -2, stop = 2, num = npoints, endpoint = True)
psiMRL = np.logspace(start = -2, stop = 2, num = npoints, endpoint = True)

cblModel = VanGenuchten(thR = cblthR, thS = cblthS, aVG = cblaVG, nVG = cblnVG, mVG = 1-1/cblnVG,
                        lVG = cbllVG, ksat = cblksat, psi = psiCBL)
mrlModel = VanGenuchten(thR = mrlthR, thS = mrlthS, aVG = mrlaVG, nVG = mrlnVG, mVG = 1-1/mrlnVG,
                        lVG = mrllVG, ksat = mrlksat, psi = psiMRL)
```

### Calculer le modèle de Kisch

```python
unit_flow = 1e-8

# CBL profile
profileCBL = VerticalFlow(model = cblModel, q = unit_flow, psiMin = 1e-4, zMin = 0)
profileCBL.kisch()

# MRL profile
profileMRL = VerticalFlow(model = mrlModel, q = unit_flow,
                          psiMin = np.max(profileCBL.kisch()[0]), zMin = 0.5, perc = 1)
profileMRL.kisch()

print ('La succion maximale obtenur dans la CBL pour un débit de ', unit_flow, 'm/s est de', profileCBL.psiMax, 'm.')
print ('La succion maximale obtenur dans la MRL pour un débit de ', unit_flow, 'm/s est de', profileMRL.psiMax, 'm.')
```

    La succion maximale obtenur dans la CBL pour un débit de  1e-08 m/s est de 0.146582685679 m.
    La succion maximale obtenur dans la MRL pour un débit de  1e-08 m/s est de 0.235467174528 m.

### Graphiques des fonctions hydrauliques

```python
data = pd.DataFrame.from_items([('psi', cblModel.wrc()[0]),
                                ('theta', cblModel.wrc()[1])])
fontSize = 16
fig1, axes = plt.subplots(nrows=2, ncols=1, figsize=(6, 12))
axes[0].set_axisbelow(True)
axes[0].tick_params(labelsize = fontSize * 0.8)
axes[0].grid(b=True, which='major', color='#eeeeee', linestyle='-')
axes[0].grid(b=True, which='minor', color='#e9e9e9', linestyle='-')
axes[0].set_ylabel(r'$\theta (m^3/m^3)$', fontsize = fontSize)
axes[0].set_xscale('log')
axes[0].set_xticks([0.1, 1, 10, 100, 1000])
axes[0].get_xaxis().set_major_formatter(matplotlib.ticker.ScalarFormatter())
axes[0].plot(cblModel.wrc()[0] * 9.807, cblModel.wrc()[1], linestyle='-', color = 'black',
             linewidth=2, label="CBL material")
axes[0].plot(mrlModel.wrc()[0] * 9.807, mrlModel.wrc()[1], linestyle='-', color = 'black',
             label="MRL material")
legend = axes[0].legend(fontsize = fontSize * 0.8)
frame = legend.get_frame()
frame.set_facecolor('1.00')

axes[1].set_axisbelow(True)
axes[1].tick_params(labelsize = fontSize * 0.8)
axes[1].grid(b=True, which='major', color='#eeeeee', linestyle='-')
axes[1].grid(b=True, which='minor', color='#e9e9e9', linestyle='-')
axes[1].set_xlabel(r'$s (kPa)$', fontsize = fontSize)
axes[1].set_ylabel(r'$k (m/s)$', fontsize = fontSize)
axes[1].set_xscale('log')
axes[1].set_xticks([0.1, 1, 10, 100, 1000])
axes[1].set_yscale('log')
axes[1].set_ylim([1e-16, 1e-2])
axes[1].get_xaxis().set_major_formatter(matplotlib.ticker.ScalarFormatter())

# kFn
axes[1].plot(cblModel.hcf()[0] * 9.807, cblModel.hcf()[1], linestyle='-', linewidth = 2, color = 'black', label="CBL material")
axes[1].plot(mrlModel.hcf()[0] * 9.807, mrlModel.hcf()[1], linestyle='-', color = 'black', label="MRL material")

# Model
axes[1].plot([0.1, 1000], [unit_flow, unit_flow], # profileMRL.psiMax
             linestyle='--', color = 'black', label="Unit flow") #  unit flow
axes[1].plot([profileCBL.psiMax * 9.807, profileCBL.psiMax * 9.807], [unit_flow, 1e-16],
             linestyle='-.', color = 'black', label=r'$s_{CBL_{max}}$')
axes[1].plot([profileMRL.psiMax * 9.807, profileMRL.psiMax * 9.807], [unit_flow, 1e-16],
             linestyle=':', color = 'black', label=r'$s_{MRL_{max}}$')
axes[1].text(profileCBL.psiMax * 9.807 - 0.9, 5e-9, r"$s_{max_{CBL}}$",
             fontsize = fontSize * 0.8, color="black", rotation = "vertical")
axes[1].text(profileMRL.psiMax * 9.807 + 1.0, 5e-9, r"$s_{max_{MRL}}$",
             fontsize = fontSize * 0.8, color="black", rotation = "vertical")

legend = axes[1].legend(fontsize = fontSize * 0.8)
frame = legend.get_frame()
frame.set_facecolor('1.00')
#fig1.savefig('fig1.svg')
```

### Graphiques des profils

```python
fig2, axes = plt.subplots(nrows=1, ncols=3, figsize=(16,6))

# Suction profile
axes[0].set_axisbelow(True)
axes[0].tick_params(labelsize = fontSize * 0.8)

axes[0].xaxis.set_major_locator(plticker.MultipleLocator(base=1.0))

axes[0].grid(b=True, which='major', color='#eeeeee', linestyle='-')
axes[0].grid(b=True, which='minor', color='#e9e9e9', linestyle='-')
axes[0].set_xlabel(r'$s (kPa)$', fontsize = fontSize)
axes[0].set_ylabel(r'$Elevation (m)$', fontsize = fontSize)
axes[0].plot(np.append(profileCBL.kisch()[0], profileMRL.kisch()[0]) * 9.807,
             np.append(profileCBL.kisch()[1], profileMRL.kisch()[1]),
             linestyle='-', color = 'black')
axes[0].plot([profileCBL.psiMax * 9.807, profileCBL.psiMax * 9.807],
             [0, 0.5],
             linestyle='-.', color = 'black')
axes[0].plot([profileMRL.psiMax * 9.807,profileMRL.psiMax * 9.807],
             [0, max(profileMRL.kisch()[1])],
             linestyle=':', color = 'black')
axes[0].plot([0, 4.0], [0.5, 0.5], linestyle = "--", color = "#aaaaaa")

# Water content profile
psiInterfaceCBL = np.append(profileCBL.kisch()[0], profileCBL.psiMax)
zInterfaceCBL = np.append(profileCBL.kisch()[1], 0.49999)
cblNew = VanGenuchten(thR = cblthR, thS = cblthS, aVG = cblaVG, nVG = cblnVG, mVG = 1-1/cblnVG,
                      lVG = cbllVG, ksat = cblksat, psi = psiInterfaceCBL)
mrlNew = VanGenuchten(thR = mrlthR, thS = mrlthS, aVG = mrlaVG, nVG = mrlnVG, mVG = 1-1/mrlnVG,
                      lVG = mrllVG, ksat = mrlksat, psi = profileMRL.kisch()[0])

axes[1].set_axisbelow(True)
axes[1].tick_params(labelsize = fontSize * 0.8)
axes[1].grid(b=True, which='major', color='#eeeeee', linestyle='-')
axes[1].grid(b=True, which='minor', color='#e9e9e9', linestyle='-')
axes[1].set_xlabel(r'$\theta (m^3/m^3)$', fontsize = fontSize)
axes[1].plot(np.append(cblNew.wrc()[1], mrlNew.wrc()[1]),
             np.append(zInterfaceCBL, profileMRL.kisch()[1]),
             linestyle='-', color = 'black')
axes[1].plot([0, 0.5], [0.5, 0.5], linestyle = "--", color = "#aaaaaa")

# Hydraulic conductivity profile
axes[2].set_axisbelow(True)
axes[2].tick_params(labelsize = fontSize * 0.8)
axes[2].grid(b=True, which='major', color='#eeeeee', linestyle='-')
axes[2].grid(b=True, which='minor', color='#e9e9e9', linestyle='-')
axes[2].set_xlabel(r'$k (m/s)$', fontsize = fontSize)
axes[2].set_xscale('log')
axes[2].set_xlim([1e-9, 1e-4])
axes[2].plot(np.append(cblNew.hcf()[1], mrlNew.hcf()[1]),
             np.append(zInterfaceCBL, profileMRL.kisch()[1]),
             linestyle='-', color = 'black')
axes[2].plot([unit_flow, unit_flow], [0, max(profileMRL.kisch()[1])], linestyle = "--", color = "black")
axes[2].plot([0, 0.5], [1e-3, 0.5], linestyle = "--", color = "#aaaaaa")
#fig2.savefig('fig2.svg')
```

## Drainage latéral dans une barrière capillaire inclinée

Ross (1990) décrit le modèle suivant.

\begin{align}
Q_{max} = \tan(\phi) \int_{\psi_{CBC}}^{\psi_{CRC}} k(\psi) d\psi \\
L = \frac {Q_{max}}{q}
\end{align}

```python
def k_vanGenuchten(x, aVG, nVG, mVG, ksat, lVG = 0.5):
        k = ksat * ((1 - ((aVG * x)**(nVG * mVG)) * \
        ((1 + ((aVG * x)**nVG))**(-mVG)))**2) / \
        ((1 + ((aVG * x)**nVG))**(mVG * lVG))
        return(k)
```

### Capacité et longueur de transfert

```python
pente = 0.25

Qmax = pente * quad(k_vanGenuchten, profileCBL.psiMax, profileMRL.psiMax, args=(mrlaVG, mrlnVG, mrlmVG, mrlksat, mrllVG))[0]
L = Qmax / unit_flow

print ('Le débit de transfert maximal dans la barrière capillaire est de', Qmax, 'm²/s.')
print ('La longueur de transfert maximale dans la barrière capillaire est de', L, 'm.')
```

    Le débit de transfert maximal dans la barrière capillaire est de 2.833509163149944e-10 m²/s.
    La longueur de transfert maximale dans la barrière capillaire est de 0.02833509163149944 m.

### Épaisseur minimale de la couche de bris capillaire (CBC ou CBL)

```python
profileCBL.psiMax
```

    0.14658268567879312

### Épaisseur de la couche de rétention capillaire (CRC ou MRL)

```python
mrlPsi = np.linspace(start = profileCBL.psiMax, stop = profileMRL.psiMax, num = 20, endpoint = True)
mrlTh = mrlPsi - profileCBL.psiMax
QmaxTh = np.array([])
for i in arange(0, len(mrlPsi)):
    QmaxTh = np.append(QmaxTh, pente * quad(k_vanGenuchten, profileCBL.psiMax, mrlPsi[i], args=(mrlaVG, mrlnVG, mrlmVG, mrlksat, mrllVG))[0])

LTh = QmaxTh / unit_flow
QmaxTh
```

    array([  0.00000000e+00,   1.87550598e-11,   3.69779707e-11,
             5.46919598e-11,   7.19187957e-11,   8.86789120e-11,
             1.04991518e-10,   1.20874698e-10,   1.36345499e-10,
             1.51420013e-10,   1.66113443e-10,   1.80440178e-10,
             1.94413840e-10,   2.08047350e-10,   2.21352964e-10,
             2.34342330e-10,   2.47026520e-10,   2.59416069e-10,
             2.71521013e-10,   2.83350916e-10])

```python
plot(mrlTh, LTh, color = 'black')
xlabel('MRL thickness (m)', fontsize = fontSize)
ylabel('Diversion length (m)', fontsize = fontSize)
grid()
```


 Multiplicadores de Lagrange

# ¿Para qué sirven?

Los multiplicadores de Lagarange es una técnica para encontrar los máximos y minímos de una función $F(x,y,z)$, donde los valores de las variables $x$, $y$ y $z$ deben satisfacer una ecuación de la forma $$ g(x,y,z) = 0. $$

Ésta ecuación se conoce como restricción del problema. A continuación presentamos el método.

Para encontrar los máximos y mínimos, vamos a seguir los siguientes pasos.

* Construir una nueva función denotada por $L$ y definida por:

	$$ L(x,y,z,\lambda) = F(x,y,z) - \lambda \, g(x,y,z) .     $$

 La función $L$ se llama función *Lagrangiana* y $\lambda$ se llama **Multiplicador de Lagrange**.

* Calcule todas las derivadas parciales de primer orden de $L$.

* Encuentre la solución del siguiente sistema de ecuaciones

 \begin{align*}
	L_x &= 0 \\
	L_y &= 0 \\
	L_z &= 0 \\
	L_{\lambda} &= 0.
 \end{align*}

* Sea $x=a$, $y=b$, $z=c$ y $\lambda=\lambda_0$, la solución del sistema de ecuaciones en el paso anterior. El punto $(a,b,c)$ se llama **punto crítico** de la función $L$.

* Los puntos críticos de $L$ son los posibles candidatos para ser máximos o mínimos de la función $F$.

* Evalue $F$ en todos los puntos críticos. El valor más grande de estos valores es el valor máximo de $F$, el más pequeño es el   valor mínimo de $F$.

* La función $F$ y la restricción $g$ también pueden ser funciones de dos variables y el metodo funciona de igual manera.

## Ejemplo

Calcular los valores máximo y minimo de la función $F(x,y,z) = 2x + 6y + 10z$, sujeta a la restricción $ x^2 + y^2 + z^2 =35.$

**Solución**

Primero cargamos los paquetes para hacer calculo simbólicos

La función $g$ de la restricción esta dada por $g(x,y,z) = x^2 + y^2 + z^2 - 35$, luego la función lagrangiana está dada por

$$  L =  2x + 6y + 10z - \lambda (x^2 + y^2 + z^2 - 35) $$ y seguimos el método, utilizando Python.

```python
from sympy import *    # cargar paquetes para hacer calculos matemáticos
init_printing(use_latex=True)
```

```python
x,y,z,l =symbols("x,y,z,l")  # Creación de las variables de f y g, reemplazamos el multiplicador de Lagrange por l.
```

```python
F = 2*x + 6*y + 10*z   # Función objetivo
F
```

```python
g = x**2 + y**2 + z**2 - 35  # Restricción
g
```

```python
L = F - l*g # Función lagrangiana
L
```

```python
Lx = diff(L,x)  # Derivadas parciales de la
Ly = diff(L,y)  # función lagrangiana con
Lz = diff(L,z)  # respecto a las variables x,y,z,l
Ll = diff(L,l)
```

```python
PC = solve([Lx,Ly,Lz,Ll],[x,y,z,l])  # Solución del sitema de ecuaciones del
PC                                  # método de multilicadores de Lagrange
```

Debemos descartar el ultimo valor de cada punto critico, el cual corresponde al valor de $\lambda$.

```python
v1 = F.subs([(x,PC[0][0]),(y,PC[0][1]),(z,PC[0][2])]) # Evaluación del  punto crítico x=-1  y=-3   z=-5
v1
```

```python
v2 = F.subs([(x,PC[1][0]),(y,PC[1][1]),(z,PC[1][2])]) # Evaluación del  punto crítico x=1  y=3   z=5
v2
```

Concluimos que la función $F(x,y) = 2x + 6y + 10z$, sujeta a la restricción $ x^2 + y^2 + z^2 =35$ tiene un máximo cuando
$x=-1$, $y=-3$ y $z=-5$ y tiene mínimo cuando $x=1$, $y=3$ y $z=5$. El valor máximo del problema  es $70$ y el valor mínimo es $-70$.

## Ejemplo (Libro de Arya)

El costo de producir $x$ modelos regulares y $y$ modelos de lujo del producto de una
empresa está dado por la función conjunta de costo

$$ C(x, y) = x^2 + 1.5y^2 + 300. $$

¿Cuántas unidades de cada tipo deben producirse para minimizar los costos totales, si la empresa
decide producir un total de 200 unidades?

**Solución**

El hecho de que la empresa produce unicamente un total de 200 unidades, siginifica que se debe cumplir la ecuación
$x + y = 200$. Por lo tanto debemos que encontrar el valor mínimo de la función de costos $ C(x, y) = x^2 + 1.5y^2 + 300$,
sujeta a la restricción $x + y - 200 = 0$.

```python
C = x**2 + 1.5*y**2 + 300   # Función objetivo
C
```

```python
g = x + y - 200   # restricción
g
```

```python
L = C - l*g   # Función lagrangiana
L
```

```python
Lx = diff(L,x)  # Derivadas parciales de la
Ly = diff(L,y)  # función lagrangiana con
Ll = diff(L,l)  # respecto a las variables x,y,l
```

```python
PC = solve([Lx,Ly,Ll],[x,y,l])  # Solución del sitema de ecuaciones del
PC                                  # método de multilicadores de Lagrange
```

El punto crítico está dado por $x = 120$   $y=80$

```python
v1 = C.subs([(x,120),(y,80)]) # Evaluación del  punto crítico x=120  y=80
v1
```

Ahora comparemos este valor con la imagen de cualquier otro que satisfaga la restricción, por ejemplo
con $x = 150$,  $y = 50$, claramente estos valores satisfacen la restricción.

```python
v2 = C.subs([(x,150),(y,50)]) # Evaluación de un  punto arbitrario que satisface la restricción x=120  y=80
v2
```

```python
v3 = C.subs([(x,100),(y,100)]) # Evaluación de un  punto arbitrario que satisface la restricción x=100  y=100
v3
```

Observamos que cuando $x = 120$ y $y = 80$ la función de costos se hace minima.

Por lo tanto la empresa debe producir 120 modelos regulares y 80 modelos de lujo para obtener el costo mínimo.

# Resumen del proceso:

1. Obtener la función lagrangiana.
2. Derivar parcialmente.
3. Encontrar el punto crítico.
4. Evaluar el punto crítico.
5. Evaluar otro punto que cumpla la restricción.
6. Clasificar el punto crítico.

## Ejercicios

Mediante el método de multiplicadores de Lagrange,
determine los puntos críticos de f sujetos a las restricciones
dadas.

* $f(x,y) = x^2 + y^2$,

   $2x +3y = 7$.

* $f(x,y,z) = x^2+ y^2 + z^2  $

   $2x +3y + 4z = 29$

*  $f(x,y,z) = xyz  $

   $xy +yz + 2xz = 24$

```python
f=x*y*z
f
```

```python
g=x*y+y*z+2*x*z-24
g
```

```python
L=f+l*g
L
```

```python
dLx=diff(L,x)
dLx
```

```python
dLy=diff(L,y)
dLy
```

```python
dLz=diff(L,z)
dLz
```

```python
dLl=diff(L,l)
dLl
```

```python
solve([dLx,dLy,dLz,dLl],[x,y,z,l])
```

solve([ecuaciones,separadas,por,comas],[variables,separadas,por,comas])

```python
f.subs([(x,-2),(y,-4),(z,-2)])
```

```python
f.subs([(x,2),(y,4),(z,2)])
```

$(-2,-4,-2)$ es un mínimo

$(2,4,2)$ es un máximo

```python
L,K,l =symbols("L,K,l")
```

```python
#Función objetivo es la produción
P=80*(L**(3/4))*(K**(1/4))
P
```

$$60L+200K=40000$$
hay que igualar a cero
$$60L+200K-40000=0$$

```python
g=60*L+200*K-40000 #Restricción
g
```

```python
F=P-l*g
F
```

```python
FK=diff(F,K)
FL=diff(F,L)
Fl=diff(F,l)
```

```python
PC=solve([FK,FL,Fl],[K,L,l])
PC
```

Como hay un único punto crítico, hay que buscar un punto de prueba:

```python
g.subs(L,400)
```

```python
solve(g.subs(L,400),K)
```

El punto de prueba es $K=80$ y $L=400$

```python
P.subs([(K,50),(L,500)])
```

```python
P.subs([(K,80),(L,400)])
```

El punto crítico $K=50$ y $L=500$ es un máximo, y la producción será de $22.493$

```python

```


# Homework 2

**For exercises in the week 20-25.11.19**

**Points: 6 + 1b**

Please solve the problems at home and bring to class a [declaration form](http://ii.uni.wroc.pl/~jmi/Dydaktyka/misc/kupony-klasyczne.pdf) to indicate which problems you are willing to present on the blackboard.

## Problem 1 [1p]

Let $(x^{(i)},y^{(i)})$ be a data sample with $x^{(i)}\in\mathbb{R}^D$, $y^{(i)}\in\mathbb{R}$. Let $\Theta \in\mathbb{R}^D$ a parameter vector.

Find the closed form solution $\Theta^*$ to

$$
\min_\Theta \left(\frac{1}{2}\sum_i (\Theta^Tx^{(i)} - y^{(i)})^2 + \frac{\lambda}{2}\sum_{d=1}^D \Theta_d^2\right).
$$

## Problem 2 [1p]
Let $v\in\mathbb{R}^D$ be a vector. Define the gradient of $f(v)\in\mathbb{R}$ with respect to $v$ to be $\frac{\partial f}{\partial v} = \left[\frac{\partial f(v)}{\partial v_1}, \frac{\partial f(v)}{\partial v_2}, ..., \frac{\partial f(v)}{\partial v_D}\right]$

Find the following functions' gradients with respect to vector $[x, y, z]^T$:
1. $f_1([x, y, z]^T) = x + y$
2. $f_2([x, y, z]^T) = xy$
3. $f_3([x, y, z]^T) = x^2y^2$
4. $f_4([x, y, z]^T) = (x + y)^2$
5. $f_5([x, y, z]^T) = x^4 + x^2 y z + x y^2 z + z^4$
6. $f_6([x, y, z]^T) = e^{x + 2y}$
7. $f_7([x, y, z]^T) = \frac{1}{x y^2}$
8. $f_8([x, y, z]^T) = ax + by + c$
9. $f_9([x, y, z]^T) = \tanh(ax + by + c)$

## Problem 3 [0.5p]

Find the following functions' gradients or Jacobians with respect to vector $\mathbf{x}$, where $\mathbf{x}, \mathbf{b} \in \mathbb{R}^{n}$, $\mathbf{W} \in \mathbb{R}^{n \times n}$:

1. $\mathbf{W} \mathbf{x} + \mathbf{b}$
2. $\mathbf{x}^T \mathbf{W} \mathbf{x}$,

## Problem 4 [1p]

Find the derivative of
$-\log(S(\mathbf{x})_j)$, where $S$ is the
    softmax function
    (https://en.wikipedia.org/wiki/Softmax_function) and we are
    interested in the derivative over the $j$-th output of the
    Softmax.

## Problem 5 [0.5p]

Consider a dataset with 400 examples of class C1 and 400 of class C2.
Let tree A have 2 leaves with class distributions:

| Tree A   | C1    | C2  |
|----------|-------|-----|
| Leaf 1   | 100   | 300 |
| Leaf 2   | 300   | 100 |

and let tree B have 2 leaves with class distribution:

| Tree B   | C1    | C2  |
|----------|-------|-----|
| Leaf 1   | 200   | 400 |
| Leaf 2   | 200   |   0 |

What is the misclassification rate for both trees? Which tree is more pure according to Gini or Infogain?

## Problem 6 [1p]

Consider regresion problem, with $M$ predictors $h_m(x)$ trained to aproximate a target $y$. Define the error to be $\epsilon_m(x) = h_m(x) - y$.

Suppose you train $M$ independent classifiers with average least squares error
$$
E_{AV} = \frac{1}{M}\sum_{m=1}^M \mathbb{E}_{x}[\epsilon_m(x)^2].
$$

Further assume that the errors have zero mean and are uncorrelated:
$$
\mathbb{E}_{x}[\epsilon_m(x)] = 0\qquad\text{ and }\qquad\mathbb{E}_{x}[\epsilon_m(x)\epsilon_l(x)] = 0\text{ for } m \neq l
$$

Let the mean predictor be
$$
h_M(x) = \frac{1}{M}h_m(x).
$$

What is the average error of $h_M(x)$?

## Problem 7 [1p]

Suppose you work on a binary classification problem and train 3 weak classifiers. You combine their prediction by voting.

Can the training error rate of the voting ensemble smaller that the error rate of the individual weak predictors? Can it be larger? Show an example or prove infeasibility.

## Problem 8 [1 bonus point]

While on a walk, you notice that a locomotive has the serial number 50. Assuming that all locomotives used by PKP (the Polish railroad operator) are numbered using consecutive natural numbers, what is your estimate of $N$ the total number of locomotives operated by PKP?

Tell why the Maximum Likelihood principle may not yield satisfactory results.

Use the Bayesian approach to find the posterior distribution over
  the number of locomotives. Then compute the expected count of
  locomotives. For the prior use the power law:
  \begin{equation}
  p(N) =  \frac{1}{N^\alpha}\frac{1}{\zeta(\alpha,1)},
  \end{equation}
  where the $\zeta(s,q)=\sum_{n=0}^{\infty}\frac{1}{(q+n)^s}$ is the
  Hurwitz Zeta function
  (https://en.wikipedia.org/wiki/Hurwitz_zeta_function)
  available in Python as `scipy.special.zeta`. The use of the
  power law is motivated by the observation that the frequency of
  occurrence of a company is inversely proportional to its size (see
  also: R.L. Axtell, Zipf distribution of US firm sizes
  https://www.sciencemag.org/content/293/5536/1818).

  How would your estimate change after seeing 5 locomotives, with the
  biggest serial number among them being 50?

  **Note**: During the Second World War, a similar problem was
  encountered while trying to estimate the total German tank
  production from the serial numbers of captured machines. The
  statistical estimates were the most precise!


# Sufficient statistics for online linear regression

First, I need to recreate the data generating function from [here](https://github.com/probml/pmtk3/blob/master/matlabTools/stats/polyDataMake.m) in Python. See the code for `plot_xy`, `plot_abline`, and `SimpleOnlineLinearRegressor` on [GitHub](https://github.com/ppham27/MLaPP-solutions/blob/master/chap07/linreg.py).

```python
%matplotlib inline

import numpy as np
from scipy import stats
import matplotlib.pyplot as plt
import pandas as pd
from linreg import *

np.random.seed(2016)

def make_data(N):
    X = np.linspace(0, 20, N)
    Y = stats.norm.rvs(size=N, loc=-1.5*X + X*X/9, scale=2)
    return X, Y

X, Y = make_data(21)
print(np.column_stack((X,Y)))
plot_xy(X, Y)
plt.show()
```

Data is not really linear, but let's just do what the exercise tells us to do. Thus, our model is

\begin{equation}
y_i \sim \mathcal{N}(w_0 + w_1x_i, \sigma^2),
\end{equation}

or written in vector notation,

\begin{equation}
\mathbf{y} \sim \mathcal{N}\left(w_0\mathbf{1} + w_1\mathbf{x}, \sigma^2I\right).
\end{equation}

Thus, we have that

\begin{align}
p(\mathbf{y} \mid w_0,w_1,\sigma^2,\mathbf{x}) &= \prod_{i=1}^N\frac{1}{\sqrt{2\pi\sigma^2}}\exp\left(-\frac{1}{2\sigma^2}\left(y_i - w_0 - w_1x_i\right)^2\right) \\
l(w_0,w_1,\sigma^2) = \log p(\mathbf{y} \mid w_0,w_1,\sigma^2,\mathbf{x}) &= -\frac{N}{2}\log(2\pi) - \frac{N}{2}\log(\sigma^2) - \frac{1}{2\sigma^2}\sum_{i=1}^N\left(y_i - w_0 - w_1x_i\right)^2.
\end{align}

Let us try to maximize the log-likelihood. We first solve for $w_0$.

\begin{align}
\frac{\partial{l}}{\partial w_0} = \frac{1}{\sigma^2}\sum_{i=1}^N \left(y_i - w_0 - w_1x_i\right)
= \frac{1}{\sigma^2}\left(-Nw_0 + \sum_{i=1}^N y_i - w_1 \sum_{i=1}^Nx_i\right).
\end{align}

Setting $\frac{\partial{l}}{\partial w_0} = 0$ and solving for $w_0$, we find that

\begin{equation}
\hat{w}_0 = \frac{\sum_{i=1}^N y_i}{N} - \hat{w}_1\frac{\sum_{i=1}^N x_i}{N} = \bar{y} - \hat{w}_1\bar{x}
\end{equation}

Next, we solve for $w_1$. Taking the derivative, we have

\begin{align}
\frac{\partial{l}}{\partial w_1} = \frac{1}{\sigma^2}\sum_{i=1}^N x_i\left(y_i - w_0 - w_1x_i\right)
= \frac{1}{\sigma^2}\sum_{i=1}^N\left(x_iy_i - w_0x_i - w_1x_i^2\right).
\end{align}

Setting $frac{\partial{l}}{\partial w_1} = 0$ and substituting $\hat{w}_0$ for $w_0$, we have that

\begin{align}
0 &= \frac{1}{\sigma^2}\sum_{i=1}^N\left(x_iy_i - (\bar{y} - \hat{w}_1\bar{x})x_i - \hat{w}_1x_i^2\right) \\
&= \sum_{i=1}^N\left(x_iy_i - x_i\bar{y}\right) -\hat{w}_1\sum_{i=1}^N\left(x_i^2 - x_i\bar{x}\right).
\end{align}

Since $\sum_{i=1}^N x_i = N\bar{x}$, we have that

\begin{align}
0 &= \sum_{i=1}^N\left(x_iy_i - \bar{x}\bar{y}\right) -\hat{w}_1\sum_{i=1}^N\left(x_i^2 - \bar{x}^2\right) \\
\hat{w}_1 &= \frac{\sum_{i=1}^N\left(x_iy_i - \bar{x}\bar{y}\right)}{\sum_{i=1}^N\left(x_i^2 - \bar{x}^2\right)}
= \frac{\sum_{i=1}^N\left(x_iy_i - x_i\bar{y} -\bar{x}y_i + \bar{x}\bar{y}\right)}{\sum_{i=1}^N\left(x_i^2 - 2x_i\bar{x} + \bar{x}^2\right)} \\
&= \frac{\sum_{i=1}(x_i - \bar{x})(y_i-\bar{y})}{\sum_{i=1}(x_i - \bar{x})^2}
= \frac{\frac{1}{N}\sum_{i=1}(x_i - \bar{x})(y_i-\bar{y})}{\frac{1}{N}\sum_{i=1}(x_i - \bar{x})^2},
\end{align}

which is just the MLE for the covariance of $X$ and $Y$ over the variance of $X$. This can also be written as

\begin{equation}
\hat{w}_1 = \frac{\frac{1}{N}\sum_{i=1}(x_i - \bar{x})(y_i-\bar{y})}{\frac{1}{N}\sum_{i=1}(x_i - \bar{x})^2}
= \frac{\sum_{i=1}^N x_iy_i - \frac{1}{N}\left(\sum_{i=1}^Nx_i\right)\left(\sum_{i=1}^Ny_i\right)}{\sum_{i=1}^N x_i^2 - \frac{1}{N}\left(\sum_{i=1}^Nx_i\right)^2}.
\end{equation}

Finally, solving for $\sigma^2$, we have that

\begin{align}
\frac{\partial{l}}{\partial w_1} = -\frac{N}{2\sigma^2} + \frac{1}{2(\sigma^2)^2}\sum_{i=1}^N\left(y_i - \left(w_0 +w_1x_i\right)\right)^2.
\end{align}

Setting this equal to $0$, substituting for $w_0$ and $w_1$, we have that

\begin{align}
\hat{\sigma}^2 &= \frac{1}{N}\sum_{i=1}^N\left(y_i - \left(\hat{w}_0 +\hat{w}_1x_i\right)\right)^2
= \frac{1}{N}\sum_{i=1}^N\left(y_i^2 - 2y_i\left(\hat{w}_0 +\hat{w}_1x_i\right) + \left(\hat{w}_0 +\hat{w}_1x_i\right)^2\right) \\
&= \hat{w}_0^2 + \frac{1}{N}\left(\sum_{i=1}^Ny_i^2 - 2\hat{w}_0\sum_{i=1}^Ny_i - 2\hat{w}_1\sum_{i=1}^N x_iy_i + 2\hat{w}_0\hat{w}_1\sum_{i=1}^N x_i + \hat{w}_1^2\sum_{i=1}^N x_i^2\right).
\end{align}

Thus, our sufficient statistics are

\begin{equation}
\left(N, \sum_{i=1}^N x_i, \sum_{i=1}^N y_i,\sum_{i=1}^N x_i^2, \sum_{i=1}^N y_i^2, \sum_{i=1}^N x_iy_i\right).
\end{equation}

```python
linReg = SimpleOnlineLinearRegressor()
linReg.fit(X, Y)
## visualize model
plot_xy(X, Y)
plot_abline(linReg.get_params()['slope'], linReg.get_params()['intercept'],
            np.min(X) - 1, np.max(X) + 1,
            ax=plt.gca())
plt.title("Training data with best fit line")
plt.show()
print(linReg.get_params())
```

Now, let's verify that the online version comes to the same numbers.

```python
onlineLinReg = SimpleOnlineLinearRegressor()
w_estimates = pd.DataFrame(index=np.arange(2,22), columns=['w0_est', 'w1_est', 'sigma2'], dtype=np.float64)
for i in range(len(Y)):
    onlineLinReg.partial_fit(X[i], Y[i])
    if i >= 1:
        w_estimates.loc[i + 1] = {'w0_est': onlineLinReg.get_params()['intercept'],
                                  'w1_est': onlineLinReg.get_params()['slope'],
                                  'sigma2': onlineLinReg.get_params()['variance']}
print(w_estimates)
print(onlineLinReg.get_params())
plt.figure(figsize=(12,8))
plt.plot(w_estimates.index, w_estimates['w0_est'], 'o',
         markeredgecolor='black', markerfacecolor='none', markeredgewidth=1,
         label='Intercept estimate')
plt.plot(w_estimates.index, w_estimates['w1_est'], 'o',
         markeredgecolor='red', markerfacecolor='none', markeredgewidth=1,
         label='Slope estimate')
plt.grid()
plt.ylabel('Estimate')
plt.xlabel('# of data points')
plt.title('Online Linear Regression Estimates')
plt.hlines(onlineLinReg.get_params()['intercept'], xmin=np.min(X), xmax=np.max(X) + 5, linestyle='--',
           label='Final intercept estimate')
plt.hlines(onlineLinReg.get_params()['slope'], xmin=np.min(X), xmax=np.max(X) + 5, linestyle='--', color='red',
           label='Final slope estimate')
plt.legend(loc='center left', bbox_to_anchor=(1,0.5))
plt.show()
```


**Competing in different settings**

In this project we consider 2 firms who compete in the same duopolistic market. We will look at three possible competition forms, which are characterized by

**Cournot**

- Firms compete in quantities, and decide upon these independently and simultaneously

- Firms profit maximize given the other's choice of quantity

- Both firms have market power, thus one's decision of output affects the price of the other's goods

**Bertrand**

- Firms compete in prices, and decide upon these simultaneously and as if they were in a perfect competition setting

- We assume that consumers seek the lowest price (assuming the two products are homogenous)

- When prices are set, output is given as

$$D\left(p_{i},p_{j}\right)=\begin{cases}
D\left(p_{i}\right) & p_{i}<p_{j}\\
\frac{D\left(p_{i}\right)}{2} & p_{i}=p_{j}\\
0 & p_{i}>p_{j}
\end{cases}$$

**Stackelberg**

- Firms compete in quantities, and decide upon these sequentially.

- One firm is assumed to be the leader, and the other is the follower.

In the following we will analyse a sitaution with linear demand with $a=101$, $b=0.1$ and cost equal to 1 for all firms under all three situations.

```python
#Import packages
from scipy import optimize,arange
from numpy import array
import numpy as np
import matplotlib.pyplot as plt
import random
import sympy as sm
from math import *
%matplotlib inline
from IPython.display import Markdown, display
import pandas as pd
```

**Bertand**

First we wanna solve the case of Bertand competetion, i.e a sitatuion, where the firms compete in prices. Here we consider two companies who are selling differentiated products. The quantity each firm produces is dependent on the on it's own price $p_1$ and the competitor's price $p_2$.
$$x_1 = a - p_1 +bp_2$$
$$x_2 = a - p_2 + bp_1$$
The individual companies profits can be defined as:
$$\pi_1 = (p_1 - c_1)(a - p_1 +bp_2)$$
$$\pi_2 = (p_2 - c_2)(a - p_2 + bp_1).$$
For the following parts we assume that both companies have identical marginal costs and thus $$c_1 = c_2 = c.$$

```python
#Variables
a = sm.symbols('a')
p1 = sm.symbols('p1')
p2 = sm.symbols('p2')
c = sm.symbols('c')
b = sm.symbols('b')
alpha=sm.symbols('alpha')

#Frist we define the profit funtioncs for the two firms:
pi1 = (p1-c)*(a-p1+b*p2)
pi2 = (p2-c)*(a-p2+b*p1)
```

In order to get the equilibrium price we first have to determine the first order conditions (FOC) by maximizing the profit functions of the individual firms with repsect to the prices they offer.
$$\frac{\partial\pi_1}{\partial p_1}= a - 2p_1 + bp_2 + c=0$$
$$\frac{\partial\pi_2}{\partial p_2}= a - 2p_2 + bp_1 + c=0$$

By solving the first order conditions for the respective prices we arrive at the reaction functions.
$$p_1^{*} = \frac{a+ b*p_2 + c}{2}$$
$$p_2^{*} = \frac{a+ b*p_1 + c}{2}$$

By setting them equal, since it is the best repsonse for both firms to set their price equal to the competitior's price, we arrive at our equilibrium price:
$$p^{*} = \frac{-(a+c)}{b-2}$$

```python
#The first order conditions are then:
FOC1 = sm.diff(pi1, p1)
FOC2 = sm.diff(pi2, p2)

#Reactionsfuncions
RF1 = sm.solve(FOC1, p1)[0]
RF2 = sm.solve(FOC2, p2)[0]

#Equilibrium price
RF12 = sm.solve(FOC2,p1)[0]
EP1 = sm.solve(RF1-RF12,p2)[0]
print(EP1) #Equilibrium price both companies will charge
RF21 = sm.solve(FOC1,p2)[0]
EP2 = sm.solve(RF2-RF21,p1)[0]
```

    -(a + c)/(b - 2)

Using the lambdify feature from the sympy package we can now make a function to solve the bertrand competition for chosen values. In this case we set $a$ equal to 101, $b$ equal to 0.1 and $c$ equal to 1 (the numbers were decided on arbitrarily). This yields an equilibrium price of 53.

```python
#make it a function to solve it
bertrand = sm.lambdify((a,b,c), EP1)
price1=bertrand(101,0.1,1)
quantity1 = 101 - 0.9*bertrand(101,0.1,1)
profit1 = (bertrand(101,0.1,1)-1)*(101 - 0.9*bertrand(101,0.1,1))

#Defining a way to print the solution in a fancy way
def fancy(string):
    display(Markdown(string))

fancy(f'The equilibrium price is $p$ = {int(price1)}, the equilibrium quantity is $q$ = {int(quantity1)} and the profits for both firms are $π$ = {int(profit1)}')
```

The equilibrium price is $p$ = 53, the equilibrium quantity is $q$ = 52 and the profits for both firms are $π$ = 2775

**Stackelberg**

We now turn to the Stackelberg, i.e a sequantiacal competion in quantities, thus one company is the leader and the other is a follower.

```python
#values
p = sm.symbols('p')
x1 = sm.symbols('x1')
x2 = sm.symbols('x2')

#Giving values to our parameters
a = 101
c = 1
b = 0.1
```

Each firm chooses a quantitiy $x_{1}, x_{2}\geq0$ where the cost of production is $c_{1}=c_{2}$. Each firm set the price $p = a-b\left(x_{1}+x_{2}\right)$. The profit function for firm $i$ is then written as $\pi_{i}\left(x_{i}\right)=p\left(x_{i}\right)-c_{i}x_{i}$ Assume firm 1 is the leader and 2 is the follower. The game is solved by backward induction.

We will first define the inverse demand function $p$:

```python
#Defining price
p = a - b*(x1+x2)
```

Then, we set up the profit function for firm 1 and firm 2:

```python
#Profit function of firm 1
profit_1 = p*x1 - c*x1
profit_2 = p*x2 - c*x2
```

We will say that firm 1 decides first on its output. Firm 2 observes the decision of firm 1 and decides how to reacht. We can solve the problem backwards and find optimal output of firm 2 for each possible output level of firm 1.
It means that we maximize the profit of firm 2 by differentiating it by $x_2$.

```python
#Differentiating the profit of firm 2 by q2

diff_profit_2 = sm.diff(profit_2, x2)
diff_profit_2
```

$\displaystyle - 0.1 x_{1} - 0.2 x_{2} + 100$

```python
#Finding optimal q2 by isolating q2 from the differentiation
opt_2 = sm.solve(diff_profit_2, x2)
opt_2[0]
```

$\displaystyle 500.0 - 0.5 x_{1}$

Then, we can substitute $q_2$ into the profit function of firm 1. Thanks to that, we have a single variable function.

```python
#Substituting q2 into the profit function of firm 1
profit_1_subs = profit_1.subs(x2,opt_2[0])
profit_1_subs
```

$\displaystyle x_{1} \left(51.0 - 0.05 x_{1}\right) - x_{1}$

Once again, we need to differentiate the profit of firm 1 by $q_1$ to find the optimal value of $q_1$.

```python
#Differentiating the profit of firm 1 by q1

diff_profit_1 = sm.diff(profit_1_subs, x1)
diff_profit_1
```

$\displaystyle 50.0 - 0.1 x_{1}$

```python
#Finding the optimal value of q1

opt_1 = sm.solve(diff_profit_1, x1)
opt_1[0]
```

$\displaystyle 500.0$

Now, we can find the optimal value of $q_2$.

```python
opt_2_subs = opt_2[0].subs(x1,opt_1[0])
opt_2_subs
```

$\displaystyle 250.0$

We can also find the equilibrium market price, profit of firm 1 and profit of firm 2:

```python
#Finding equilibrium market price
p = a - b*(opt_1[0] + opt_2_subs)

#Finding profit of firm 1
profit_1 = p*opt_1[0] - c*opt_1[0]

#Finding profit of firm 2
profit_2 = p*opt_2_subs - c*opt_2_subs

fancy(f'The equilibrium price is $p$ = {int(p)}, and the profits for both firms are $π_1$ = {int(profit_1)} and $π_2$ = {int(profit_2)}')
```

The equilibrium price is $p$ = 26, and the profits for both firms are $π_1$ = 12500 and $π_2$ = 6250

We now want to make a figure that show how the beste responses relate.

```python
#Defining the x-axis
qu_1 = np.linspace(0, 1000, 1000)

#Defining the lambda functions for our best responses functions
BR2 = lambda qu_1 : 500 - qu_1/2
BR1 = lambda qu_1 : 500 - qu_1/2

#Plotting the figure
plt.figure(figsize=(15,10))
plt.plot(qu_1, BR2(qu_1), label = "Best Response of Firm 1")
plt.plot(BR1(qu_1),qu_1 , label = "Best Response of Firm 2")
plt.grid(True)
plt.xlabel('$q1$')
plt.ylabel('$q2$')
plt.title('Best response functions')
plt.legend()

#To include a quantity of 0 for 1 firm, we set the axis like this
plt.xlim(0,1000) # sets the x-axis
plt.ylim(-100,600) # Sets the y-axis

plt.show()
```

This tells that when firm chooses a quantity of 500 in the first stage, firm 2 will choose a quantity half the size of firm 1 in second stage. In a case where firm 1 chooses to produce 1000 units, firm 2 will then produce 0.

**Cournot**

For the Cournot case we are gonna set up some functions to solve the model in order to make an extention with n firms later on.

Consider the linear invese demand curve $p_{i}\left(x_{i},x_{j}\right)=a-x_{i}-bx_{j}$ where $b\in\left(0,1\right)$ is the elasticity of substitution between the goods, i.e. indicates that the goods are heterogenous for $0<b<1$.

We write the inverse demand:

```python
def price(x1,x2,a,b):
    """A function that computes prices
    args:
    x1 : quantity for firm 1
    x2 : quantity for firm 2
    a :total demand
    b : elasticity of substitution

    returns: prices
    """
    return a-b*(x1+x2)

```

Firms have constant marginal costs, where total costs are given by $c\left(x_{i,j}\right)=cx_{i,j}$

```python
def cost(x,c):
    """A function that computes total cost
    args:
    c : cost for firm i
    x : quantity for firm i
    returns :
    """
    cost = c*x
    return cost
```

Firms maximize profits given the choice of the other firm such that $$\pi\left(x_{i},x_{j}\right)=p_{i}\left(x_{i},x_{j}\right)x_{i}-c\left(x_{i}\right)$$

```python
def profit(x1,x2,c1,a,b):
    """A function that computes profit for firm 1
    args:
    x1 : quantity for firm 1
    x2 : quantity for firm 2
    c1 : cost for firm 1
    a : total demand
    b : elasticity of substitution
    """
    return price(x1,x2,a,b)*x1-cost(x1,c1)
```

In a Cournot setting, firm $i$ chooses $x_{i}$ taking $x_{j}$ as given. Thus, the Nash equilibrium for $x_{i}^{*},x_{j}^{*}$ holds for $$x_{i}^{*}=\text{arg}\underset{x_{i}}{\text{ max }}\pi\left(x_{i},x_{j}^{*}\right)\;\;\;\;\;\;\text{for each }i\neq j$$ Thus, we are able to find firm $i$ optimal production level given the choice of $x_{j}$ by creating the function 'reaction'. Hence, taking the first order condition to find the Nash equilibrium for $x_{i}^{*},x_{j}^{*}$ yields $$\frac{\partial\pi\left(x_{i},x_{j}\right)}{\partial x_{i,j}}=0$$

In this case, we also want to calculate the profit maximizing outcome, so we use the function 'brute'. This allows one to look for the global maximum of $\pi$. Likewise, we keep in mind that 'scipy optimize' is bound to minimize problems, and since we want to maximize the profit function, we then minimize $-\pi\left(x_{i},x_{j}\right)$.

```python
def reaction(x2,c1,b,a):
    """A function that computes the optimal quantity
    args
    x2 : quantity for firm 2
    c1 : cost for firm 1
    b : elasticity of substitution
    a : total demand
    """
    x1 = optimize.brute(lambda x: -profit(x,x2,c1,a,b), ((0,1,),))

    return x1[0]
```

Now, we must find an equilibrium for price and quantity given their best response. For this reason, consider the vector function $f\left(x_{i},x_{j}\right)$ which takes firm $i$'s best response $r_{i}\left(x_{j}\right)$ to $j$'s production level $x_{j}$ into consideration. We can write this as $$f\left(x_{i},x_{j}\right)=\left[\begin{array}{c}
r_{i}\left(x_{j}^{*}\right)\\
r_{j}\left(x_{i}^{*}\right)
\end{array}\right]$$

We are then looking for the quantities produced is equal to best response given what the other firm produce, i.e the point where:

$$\left(\begin{array}{c}
x_{1}^{*}\\
x_{2}^{*}
\end{array}\right)=\left(\begin{array}{c}
r_{1}(x_{2}^{*})\\
r_{2}(x_{1}^{*})
\end{array}\right)$$

Thus, we create a function 'vec_reaction' given by the difference between an array of the quantaites subtracted by the best response functions and pass $b,a, c_{i},c_{j}$ as a vector of parameters.

```python
def vec_reaction(x,param):
    """A function that computes the difference between the quantities and the beste response functions
    args
    x : quantaties
    param : list of parameters"""
    return array(x)-array([reaction(x[1],param[2],param[0],param[1]),reaction(x[0],param[3],param[0], param[1])])
```

Now, moving on to find the actual Cournot equilibrium, we are able to set the values of each paramter. Using the 'fsolve' from 'scipy.optimize' to find $x$. Also, giving 'fsolve' the inital guess of the quantaites, while passing the paramater vector 'param' as arguments 'args' to 'fsolve', this yields:

```python
param = [0.1,101,1,1]
x0 = [0.3, 0.3]

EQ = optimize.fsolve(vec_reaction, x0, args = (param))
EQ

fancy(f'The Nash equilibrium quantity for both firms are $x^* =[x_1^*,x_2^*]$ = {EQ} where the equilibrium price for both firms are $p^*$ = {price(EQ[0],EQ[1],101,0.1)} with a profit of $\pi^*$= {profit(EQ[0],EQ[1],1,101,0.1)}')

```

The Nash equilibrium quantity for both firms are $x^* =[x_1^*,x_2^*]$ = [333.33330078 333.33330078] where the equilibrium price for both firms are $p^*$ = 34.333339843749854 with a profit of $\pi^*$= 11111.11219618032

We now want to show, how the quantity produced changes, when the cost of firm 1 changes.

```python
range_c = np.linspace(1,0,20)
range_x = [optimize.fsolve(vec_reaction, x0, args = ([0.1,101, c, 1])) for c in range_c]

plt.style.use('seaborn')
plt.clf()
plt.plot(range_c,array(range_x)[:,0], label = '$x_1$')
plt.plot(range_c,array(range_x)[:,1], label = '$x_2$')
plt.xlabel('$c_1$')
plt.ylabel('$x$')
plt.legend()
plt.show
```

In the figure, we see, that as the cost of firm 1 goes up, the quantity produced by firm 1 falls, while the quantity produced by firm 2 goes up.

**Compare all three cases**

We have now tried to solve a situation with two firms competing with three different models, giving them the same parameter values. This leads to the following results:

```python
fancy(f'For Bertrand: The equilibrium price is $p$ = {int(price1)}, the equilibrium quantity is $q$ = {int(quantity1)} and the profits for both firms are $π$ = {int(profit1)}')
fancy(f'For Stackelberg: the equilibrium price is $p$ = {int(p)}, and the profits for both firms are $π_1$ = {int(profit_1)} and $π_2$ = {int(profit_2)}')
fancy(f'For Cournot: the equilibrium quantity for both firms are $x^* =[x_1^*,x_2^*]$ = {EQ} where the equilibrium price for both firms are $p^*$ = {price(EQ[0],EQ[1],101,0.1)} with a profit of $\pi^*$= {profit(EQ[0],EQ[1],1,101,0.1)}')

```

For Bertrand: The equilibrium price is $p$ = 53, the equilibrium quantity is $q$ = 52 and the profits for both firms are $π$ = 2775

For Stackelberg: the equilibrium price is $p$ = 26, and the profits for both firms are $π_1$ = 12500 and $π_2$ = 6250

For Cournot: the equilibrium quantity for both firms are $x^* =[x_1^*,x_2^*]$ = [333.33330078 333.33330078] where the equilibrium price for both firms are $p^*$ = 34.333339843749854 with a profit of $\pi^*$= 11111.11219618032

**Cournot with n> 2 firms**

In order to do an extention to one of the models, we now want to investigate the case of N > 2 firms. First we set up the situation theoretically. In the following theoretical set-up we will follow the set-up in this note, adapting it to our situation: http://people.exeter.ac.uk/dgbalken/micro07/CournotHandout.pdf (we add the posibility of different cost for each firms)

Following the notation from ealier we now add:

- Firm $i$'s output: $x_i$

- Total output in in industry: $X=x_1+x_2+...+x_n$

- The opponent's output: $x_{-i} = X-x_i$

Each firm will then max profit given the expectation of $x_{-i}$:

$$\Pi_i(x_{-i},x_i)=(p(x_{-i}+x_i)-c_i)x_i$$

as with the n = 2 case we use the invese demand funcktion in a linear case:

$$p = a-b(x_{-i}+x_i)=a-bX$$

The FOC of is then:
$$\frac{\partial \Pi_i}{\partial x_i} = \frac{\partial p}{\partial x_i} \cdot x_i + p-c_i = 0$$

where $\frac{\partial p}{\partial x_i} = -b$ giving us

$$\frac{\partial\Pi_i}{\partial x_i} : -bx_i + p-c_i = 0$$

inserting the inverse demand function:

$$\frac{\partial\Pi_i}{\partial x_i} : -bx_i + (a-b(x_{-i}+x_{i}))-c_i = 0$$

solving for $x_i$ to get the reaction function for firm $i$ given its expectation for the amount the rest of the firms produce:

$$x_i(x_{-i}) = \frac{a-c}{2b}-\frac{1}{2}x_{-i}$$

which is the case for all firms $i=1,..,n$. We have $X=x_i+x_{-i}$ so the FOC found earlier can be rewritten to:
$$ -bx_i + (a-bX-c_i) = 0$$

which for the n firms:
$$-bX + n(a-bX)-n\bar c = 0$$

where we use the average cost level: $\bar c = \frac{c_1+c_2+...+c_n}{n}$

we can now rewrite this and get the quantity produced and the price in the market:

$$bX + n(bX) = n(A-\bar c)$$

$$(n+1)bX  = n(A-\bar c)$$

Quantity is then:

$$X = \frac{n}{n+1} \frac{a-\bar c}{b}$$

Price is:
$$p = a-bX = a-b[\frac{n}{n+1} \frac{a-\bar c}{b}] = \frac{1}{n+1}a + \frac{n}{n+1}\bar c$$

Using the found functions we can now try to set a program that is able to solve cases with N firms.

```python
def Round(a):
    """A function that prints floats rounded as strings
    args:
    a : a float
    returns : """
    return str("{:.2f}".format(a))
```

First we define a function, that calculates the prices $p = \frac{1}{n+1}a + \frac{n}{n+1}\bar c $ rewriting using the definition of the average cost level gives: $p= \frac{1}{n+1}a + \frac{c_1+c_2+...+c_n}{n+1}$ which we use in our function.

```python
def price_eq(a, n, c): #Returns the equilibirum price given n firms
    """A function that computes price
    args:
    a : total demand
    n : number of firms
    c : cost of firm i
    """

    price = max((a+sum(c))/(1+n), 0)
    return price
```

We now want to define a function that returns the quantity produced for firm $i$.
this is given by:
$$x_i = \frac{a-bX-c_i}{b}=\frac{a-c_i}{b}-\frac{n}{n+1}\frac{a-\bar c}{b} = \frac{1}{n+1}\frac{a}{b}+ \frac{n(\bar c -c_i)-c_i}{(n+1)b} = \frac{1}{n+1}\frac{1}{b}[a- n(\bar c -c_i)-c_i]$$

use that $\bar c = \frac{c_1+c_2+...+c_n}{n}$
$$x_i = \frac{1}{n+1}\frac{1}{b}[a+ nc_i+(c_1+c_2+...+c_n-c_i)]$$
which we now use to define a function, that returns the quantity:

```python
def xi_eq(a, b, n, c, i): #Returns equilibrium quantity for firm i
    """A function that computes quantity for firm i
    args:
    a : total demand
    b : elasticity of substitution between the good
    n : number of firms
    c : cost of firm i
    i : the number of the firm
    Returns a float"""

    #x_i = max((1/b)*((a-n*c[i]+(sum(c)-c[i]))/(1+n)), 0)
    #if x_i < 1:
        #x_i = 0
    x_i = (1/b)*((a+(n-1)*c[i]-sum(c))/(1+n))
    return x_i
```

We can then define a function, that returns the quantity for the entire number of firms

```python
def X_eq(a, b, n, c):
    """A function that computes quantity for the total amount of firms
    args:
    a : total demand
    b : elasticity of substitution between the good
    n : number of firms
    c : cost of firm i
    """

    X = []
    for i in range(0, n, 1):
        X.append(xi_eq(a, b, n, c, i))

    return sum(X)
```

Based on the assumtion that $$c_i = \bar c = c $$ the profit for firm $i$ can be found by:
$$\Pi_{i}=(p-c)x_{i}=\left(\frac{a+nc}{n+1}-c\right)\frac{1}{n+1}\frac{1}{b}(a-c)$$

Without identical marginal costs $$c_i \neq c $$ $$\bar c \neq c$$ the profit for firm $i$ would be:
$$\Pi_{i}=(p-c_i)x_{i}=\left(\frac{a+n\bar c}{n+1}-c_i\right)\frac{1}{n+1}\frac{1}{b}(a+n(\bar c - c_i)-c_1) = \frac{1}{(n+1)^2 * b}(a+n(\bar c - c_1)-c_i)^2$$

By maximizing the previously derived profit function we find the equilibrium profit.

```python
def profit_eq(a, b, n, c, i):
    """A function that computes profit for firm i and sets profit to zero, if there is no production
    args:
    a : total demand
    b : elasticity of substitution between the good
    n : number of firms
    c : cost of firm i
    i : the number of the firm
    """

    if xi_eq(a, b, n, c, i) == 0:
        profit = 0
    else:
        profit = max((1/b)*(((a-n*c[i]+(sum(c)-c[i]))/(1+n))**2), 0)

    return profit

```

We are now ready to define a function, that can solve a general case of cournot competetion with n firms. We choose the number of firms to be drawn randomely using random.randint, while the cost for each firm will be drawn from a uniform destribution using random.uniform.

```python
def EQ_N(a, b, n, c_low, c_high):
    """A function that draws number of firms and compute quantaties, price and average profit
    args:
    a : total demand
    b : elasticity of substitution between the good
    N : the maximal number of firms that can be drawn
    c_low : the lower bound of the interval for the cost
    c_high : the upper bound of the interval for the cost
    Returns a float"""

    #draw the number of firms:
    #n = random.randint(2, N)

    #drawing of costs for each firm:
    c = []
    #for i in range(0, n, 1):
     #   c_i = random.uniform(c_low, c_high)
      #  c.append(c_i)
    np.random.seed(1986)
    c = np.random.uniform(c_low,c_high,size=n)
    #the price:
    price = price_eq(a, n, c)

    #quantities and profits
    xi_list = []
    profiti_list = []

    #the quantity for the entire number of firms
    X = X_eq(a, b, n, c)

    return X, price, c

```

First we solve with 1000 firms and use the same parameters as with the other three models:

```python
a = 101
b = 0.1
c_low = 1
c_high = 5
X,price, c_vec = EQ_N(a,b,1000,c_low,c_high)
print('\n The number of firms are 1000 ' +
                  '\n The total amount produced is ' + Round(X) +',' + ' while the price is ' + Round(price) +
                   '\n')

```

     The number of firms are 1000
     The total amount produced is 978.80, while the price is 3.12

**Quantity and pice as functions of n**

We now want to show how the amout produced depends on the number of firms n. We therefor create a list with a range for n:

```python
n_vec = list(range(100,1000))

output_vec1 = array([EQ_N(a,b,n,c_low,c_high) for n in n_vec])

fig,ax = plt.subplots()
ax.plot(n_vec,output_vec1[:,0], label = '$X$')
plt.legend()
plt.show()
```

Furthermore we can show how the price varies with number of firms **Asker**

```python
def solution(a,b,c_vec):
    n= len(c_vec) #antallet af firmaer er længden af c_vectoren
    q = n/(n+1) * (a-np.mean(c_vec))/b
    p = a-b*q
    return q,p

output_vec = array([solution(a,b,c_vec) for c_vec in output_vec1[:,2]])

plt.style.use('seaborn')
plt.clf()
plt.plot(n_vec,output_vec[:,1], label = '$price$')
plt.legend()
plt.show()
```

**Solving with python**\
Up untill now we have solved the model analytically and then used python to show the results. However the model can also be solved using python, which will be done in the following section:

er N antallet af firmaer? **Asker**

```python
N = 1000 # number of firms
np.random.seed(1986)
c_low = 1
c_high = 5
c_vec = np.random.uniform(c_low,c_high,size=N)
```

We then use the profit function: $\Pi_i = (p(x_{-i}+x_{i})-c_i)x_i$ with $p=a-b(x_{-i}+x_i)$ to find the foc with sympy:

```python

```

```python
def focs(a,b,q_vec,c_vec):
    # Using analytical FOCs from pdf-link -> kom frem til denne ved sympy!!
    return -b*q_vec+a-b*np.sum(q_vec)-c_vec
```

```python
def solve(a,b,c_vec):
    obj = lambda q_vec : focs(a,b,q_vec,c_vec)

    res = optimize.root(obj, x0 = [90 for n in range(N)])

    q_vec_star = res.x
    Q = np.sum(q_vec_star)

    # Use inverse demand to find price
    p = a- b*Q
    return Q, p
```

```python
a = 101
b = 0.1
Q,p = solve(a,b,c_vec)
print('\n The number of firms are 1000 ' +
                  '\n The total amount produced is ' + Round(Q) +',' + ' while the price is ' + Round(p) +
                   '\n')

```

     The number of firms are 1000
     The total amount produced is 978.80, while the price is 3.12

```python
#Compare with the solution from the analytical solution
X,price, c_vec = EQ_N(a,b,1000,c_low,c_high)
print('\n The number of firms are 100 ' +
                  '\n The total amount produced is ' + Round(X) +',' + ' while the price is ' + Round(price) +
                   '\n')
```

```python
n_vec = list(range(100,1000))

output_vec1 = array([solve(a,b,c_vec)for N in n_vec])

fig,ax = plt.subplots()
ax.plot(n_vec,output_vec1[:,0], label = '$X$')
plt.legend()
plt.show()
```

**Conclusion**

In this paper we examine two firms who have the possibility of competing in Bertrand, Stackelberg and Cournot. We use the same parameter values for all competition types, and find that the firms benefit the most from competing in a Cournot setting as this yields the highest total profits of 22,222. This is intuitive, as in contrast to Bertrand, the firms will undercut each other until their price is equal to their marginal costs to obtain efficiency. Therefore, this setting yields a price as in perfect competition, whereas Cournot yields an oligopoly equilibrium.

As Cournot yields the best outcome in terms of profits for two firms, we extend this setting for $n$ firms. We find that for $n\rightarrow\infty$, the average profit for a firm decreases because competing in quantities becomes more intense as the competition setting moves from oligopoly to infinitely many firms competing in quantities.


# Random Signals

*This jupyter notebook is part of a [collection of notebooks](../index.ipynb) on various topics of Digital Signal Processing. Please direct questions and suggestions to [Sascha.Spors@uni-rostock.de](mailto:Sascha.Spors@uni-rostock.de).*

## Superposition of Random Signals

The superposition of two random signals

\begin{equation}
y[k] = x[k] + n[k]
\end{equation}

is a frequently applied operation in statistical signal processing. For instance, to model a measurement procedure or communication channel as superposition of the desired signal with noise. We assume that the statistical properties of the real-valued signals $x[k]$ and $n[k]$ are known. We are interested in the statistical properties of $y[k]$, as well as the joint statistical properties between the signals and their superposition $y[k]$. It is assumed for the following that $x[k]$ and $n[k]$ are drawn from wide-sense stationary (WSS) real-valued random processes.

### Cumulative Distribution and Probability Density Function

The cumulative distribution function (CDF) $P_y(\theta)$ of $y[k]$ is given by rewriting it in terms of the joint probability density function (PDF) $p_{xn}(\theta_x, \theta_n)$

\begin{equation}
P_y(\theta) = \Pr \{ y[k] \leq \theta \} = \Pr \{ (x[k] + n[k]) \leq \theta \} =
\int\limits_{-\infty}^{\infty} \int\limits_{-\infty}^{\theta - \theta_n} p_{xn}(\theta_x, \theta_n) \; \mathrm{d}\theta_x\,\mathrm{d}\theta_n
\end{equation}

Its PDF is computed by introducing above result into the [definition](distributions.ipynb#Univariate-Probability-Density-Function) of the PDF

\begin{equation}
p_y(\theta) = \frac{\mathrm{d} P_y(\theta)}{\mathrm{d}\theta} = \int\limits_{-\infty}^{\infty} p_{xn}(\theta - \theta_n, \theta_n) \; \mathrm{d}\theta_n
\end{equation}

since the inner integral on the right hand side of $P_y(\theta)$ can be interpreted as the inverse operation to the derivation with respect to $\theta$.

An important special case is that $x[k]$ and $n[k]$ are uncorrelated. Under this assumption, the joint PDF $p_{xn}(\theta_x, \theta_n)$ can be written as $p_{xn}(\theta_x, \theta_n) = p_x(\theta_x) \cdot p_n(\theta_n)$. Introducing this into above result yields

\begin{align}
p_y(\theta) &= \int\limits_{-\infty}^{\infty} p_x(\theta - \theta_n) \cdot p_n(\theta_n) \; \mathrm{d}\theta_n \\
&= p_x(\theta) * p_n(\theta)
\end{align}

Hence, the PDF of the superposition of two uncorrelated signals is given by the convolution of the PDFs of both signals.

#### Example - PDF of a superposition of two uncorrelated signals

The following example estimates the PDF of a superposition of two uncorrelated signals drawn from random processes generating samples according to the [uniformly distributed](important_distributions.ipynb#Uniform-Distribution) white noise model with $a=0$ and $b=1$.

```python
%matplotlib inline
import numpy as np
import matplotlib.pyplot as plt

K = 100000  # length of random signals

# generate random signals
np.random.seed(2)
x = np.random.uniform(size=K)
n = np.random.uniform(size=K)
y = x + n

# plot estimated pdf
plt.figure(figsize = (10, 6))
plt.hist(y, 100, density=True)
plt.title('Estimated PDF')
plt.xlabel(r'$\theta$')
plt.ylabel(r'$\hat{p}_y(\theta)$')
plt.grid()
```

**Exercise**

* Check the result of the numerical simulation by calculating the theoretical PDF of $y[k]$
* What PDF will results if you sum up an infinite number of independent uniformly distributed random signals?

Solution: The PDF of both signals $x[k]$ and $n[k]$ is given as $p_x(\theta) = p_n(\theta) = \text{rect}(\theta - \frac{1}{2})$, according to the assumptions stated above. The PDF of $y[k] = x[k] + n[k]$ is consequently given by $p_y(\theta) = \text{rect}(\theta - \frac{1}{2}) * \text{rect}(\theta - \frac{1}{2})$. The convolution of two rectangular signals results in a triangular signal. Hence

\begin{equation}
p_y(\theta) =
\begin{cases}
1 - |\theta -1| & \text{for } 0 < \theta < 2 \\
0 & \text{otherwise}
\end{cases}
\end{equation}

The [central limit theorem](https://en.wikipedia.org/wiki/Central_limit_theorem) states that a superposition of independent random variables tends towards a normal distribution for an increasing number of superpositions. Numerical simulation for a high bus finite number of independent uniformly distributed random signals yields

```python
N = 1000  # number of independent random signals

# generate random signals
np.random.seed(2)
y = np.sum(np.random.uniform(size=(N, K)), axis=0)

# plot estimated pdf
plt.figure(figsize = (10, 6))
plt.hist(y, 100, density=True)
plt.title('Estimated PDF')
plt.xlabel(r'$\theta$')
plt.ylabel(r'$\hat{p}_y(\theta)$')
plt.grid()
```

### Linear Mean

The linear mean $\mu_y$ of the superposition is derived by introducing $y[k] = x[k] + n[k]$ into the [definition of the linear mean](ensemble_averages.ipynb#Linear-mean) and exploiting the [linearity of the expectation operator](ensemble_averages.ipynb#Properties) as

\begin{equation}
\mu_y[k] = E \{ x[k] + n[k] \} = \mu_x[k] + \mu_n[k]
\end{equation}

The linear mean of the superposition of two random signals is the superposition of its linear means.

### Auto-Correlation Function and Power Spectral Density

The ACF is computed in the same manner as the linear mean by inserting the superposition into the [definition](correlation_functions.ipynb#Auto-Correlation-Function) of the ACF and rearranging terms

\begin{align}
\varphi_{yy}[\kappa] &= E\{ y[k] \cdot y[k-\kappa] \} \\
&= E\{ (x[k] + n[k]) \cdot (x[k-\kappa] + n[k-\kappa]) \} \\
&= \varphi_{xx}[\kappa] + \varphi_{xn}[\kappa] + \varphi_{nx}[\kappa] + \varphi_{nn}[\kappa]
\end{align}

The ACF of the superposition of two random signals is given as the superposition of all auto- and cross-correlation functions (CCFs) of the two random signals. The power spectral density (PSD) is derived by performing a discrete-time Fourier transform (DTFT) of the ACF

\begin{equation}
\Phi_{yy}(\mathrm{e}^{\,\mathrm{j}\,\Omega}) = \Phi_{xx}(\mathrm{e}^{\,\mathrm{j}\,\Omega}) + \Phi_{xn}(\mathrm{e}^{\,\mathrm{j}\,\Omega}) + \Phi_{nx}(\mathrm{e}^{\,\mathrm{j}\,\Omega}) + \Phi_{nn}(\mathrm{e}^{\,\mathrm{j}\,\Omega})
\end{equation}

This can be simplified further by exploiting the symmetry property of the CCFs $\varphi_{xn}[\kappa] = \varphi_{nx}[-\kappa]$ and the DTFT for real-valued signals as

\begin{equation}
\Phi_{yy}(\mathrm{e}^{\,\mathrm{j}\,\Omega}) = \Phi_{xx}(\mathrm{e}^{\,\mathrm{j}\,\Omega}) + 2\,\Re \{ \Phi_{xn}(\mathrm{e}^{\,\mathrm{j}\,\Omega}) \} + \Phi_{nn}(\mathrm{e}^{\,\mathrm{j}\,\Omega})
\end{equation}

where $\Re \{ \cdot \}$ denotes the real part of its argument.

### Cross-Correlation Function and Cross Power Spectral Density

The CCF $\varphi_{ny}[\kappa]$ between the random signal $n[k]$ and the superposition $y[k]$ is derived again by introducing the superposition into the [definition of the CCF](correlation_functions.ipynb#Cross-Correlation-Function)

\begin{equation}
\varphi_{ny}[\kappa] = E\{ n[k] \cdot (x[k-\kappa] + n[k-\kappa]) \} = \varphi_{nx}[\kappa] + \varphi_{nn}[\kappa]
\end{equation}

It is given as the superposition of the CCF between the two random signals and the ACF of $n[k]$. The cross PSD is derived by applying the DTFT to $\varphi_{ny}[\kappa]$

\begin{equation}
\Phi_{ny}(\mathrm{e}^{\,\mathrm{j}\,\Omega}) = \Phi_{nx}(\mathrm{e}^{\,\mathrm{j}\,\Omega}) + \Phi_{nn}(\mathrm{e}^{\,\mathrm{j}\,\Omega})
\end{equation}

The CCF $\varphi_{xy}[\kappa]$ and cross PSD $\Phi_{xy}(\mathrm{e}^{\,\mathrm{j}\,\Omega})$ can be derived by exchanging the signals $n[k]$ and $x[k]$

\begin{equation}
\varphi_{xy}[\kappa] = E\{ x[k] \cdot (x[k-\kappa] + n[k-\kappa]) \} = \varphi_{xx}[\kappa] + \varphi_{xn}[\kappa]
\end{equation}

and

\begin{equation}
\Phi_{xy}(\mathrm{e}^{\,\mathrm{j}\,\Omega}) = \Phi_{xx}(\mathrm{e}^{\,\mathrm{j}\,\Omega}) + \Phi_{xn}(\mathrm{e}^{\,\mathrm{j}\,\Omega})
\end{equation}

### Additive White Gaussian Noise

In order to model the effect of distortions, it is often assumed that a random signal $x[k]$ is distorted by additive normal distributed white noise $n[k]$ resulting in the observed signal $y[k] = x[k] + n[k]$. It is furthermore assumed that the noise $n[k]$ is uncorrelated to the signal $x[k]$. This model is known as [additive white Gaussian noise](https://en.wikipedia.org/wiki/Additive_white_Gaussian_noise) (AWGN) model.

For zero-mean random processes and from the properties of the AWGN model, it follows that $\varphi_{xn}[\kappa] = \varphi_{nx}[\kappa] = 0$ and $\varphi_{nn}[\kappa] = N_0 \cdot \delta[\kappa]$. Introducing this into above results for additive random signals yields the following relations for the AWGN model

\begin{align}
\varphi_{yy}[\kappa] &= \varphi_{xx}[\kappa] + N_0 \cdot \delta[\kappa] \\
\varphi_{ny}[\kappa] &= N_0 \cdot \delta[\kappa] \\
\varphi_{xy}[\kappa] &= \varphi_{xx}[\kappa]
\end{align}

The PSDs are given as the DTFTs of the ACF/CCFs. The AWGN model is frequently assumed in communications as well as in the measurement of physical quantities to cope for background, sensor and amplifier noise.

### Example - Estimating the Period of a Noisy Periodic Signal

For the following numerical example, the disturbance of a harmonic signal $x[k] = \cos(\Omega_0 k)$ by unit variance AWGN $n[k]$ is considered. The observed signal is given as

\begin{equation}
y[k] = x[k] + n[k]
\end{equation}

The ACF $\varphi_{yy}[\kappa]$ of the compound signal, as well as the CCF between noise and signal $\varphi_{ny}[\kappa]$ is computed and plotted.

```python
N = 1024  # length of compound signals
M = 25  # period of cosine signal
K = 2*M  # maximum lag for ACF/CCF

# generate signals
x = np.cos(2*np.pi/M * np.arange(N))
np.random.seed(2)
n = np.random.normal(size=N)
# superposition of signals
y = x + n

# compute and truncate ACF of superposition
acf = 1/N * np.correlate(y, y, mode='full')
acf = acf[(len(y)-1)-K:len(y)+K]
# compute and truncate CCF of superposition and noise
ccf = 1/N * np.correlate(n, y, mode='full')
ccf = ccf[(len(y)-1)-K:len(y)+K]

# plot results
kappa = np.arange(-K, K+1)

plt.figure(figsize=(10, 10))

plt.subplot(311)
plt.stem(y, use_line_collection=True)
plt.title('Signal')
plt.xlabel(r'$k$')
plt.ylabel(r'$x[k]$')
plt.axis([0, K, -3, 3])

plt.subplot(312)
plt.stem(kappa, acf, use_line_collection=True)
plt.title('ACF of superposition')
plt.xlabel(r'$\kappa$')
plt.ylabel(r'$\varphi_{yy}[\kappa]$')
plt.axis([-K, K, -.75, 1.1*np.max(acf)])
plt.grid()

plt.subplot(313)
plt.stem(kappa, ccf, use_line_collection=True)
plt.title('CCF between noise and superposition')
plt.xlabel(r'$\kappa$')
plt.ylabel(r'$\varphi_{ny}[\kappa]$')
plt.axis([-K, K, -.2, 1.1])
plt.grid()

plt.tight_layout()
```

**Exercise**

* Derive the theoretic result for the ACF $\varphi_{xx}[\kappa]$
* Based on this, can you explain the shape of the ACF?
* Estimate the periodicity/frequency of the cosine signal from the ACF
* What conclusions can you draw from the CCF between noise and signal?

Solution: The ACF of the deterministic signal $x[k] = \cos(\Omega_0 k)$ is computed via temporal averaging. Since $x[k]$ is periodic, the ACF $\varphi_{xx}[\kappa]$ is also periodic. It is hence sufficient to perform the temporal averaging over one period $N = \frac{2 \pi}{\Omega_0}$ with $N \in \mathbb{Z}$

\begin{align}
\varphi_{xx}[\kappa] &= \frac{1}{N} \sum_{k=0}^{N-1} x[k] \cdot x[k-\kappa] \\
&= \frac{1}{N} \sum_{k=0}^{N-1} \cos(\Omega_0 k) \cdot \cos(\Omega_0 (k-\kappa))
\end{align}

Applying the [product-to-sum identity](https://en.wikipedia.org/wiki/List_of_trigonometric_identities#Product-to-sum_and_sum-to-product_identities) of the cosine function and rearranging terms yields

\begin{equation}
\varphi_{xx}[\kappa] = \frac{1}{2} \cos(\Omega_0 \kappa) + \frac{1}{2N} \sum_{k=0}^{N-1} \cos(2 \Omega_0 k - \Omega_0 \kappa)
\end{equation}

The remaining sum over the phase shifted cosine $\cos(2 \Omega_0 k - \Omega_0 \kappa)$ is zero since the summation is carried out over two full periods. The ACF of a cosine signal is then given as

\begin{equation}
\varphi_{xx}[\kappa] = \frac{1}{2} \cos(\Omega_0 \kappa)
\end{equation}

Introducing this into the relations for the AWGN model, the ACF of the superposition is

\begin{equation}
\varphi_{yy}[\kappa] = \varphi_{xx}[\kappa] + N_0 \delta[\kappa]  = \frac{1}{2} \cos(\Omega_0 \kappa) + N_0 \delta[\kappa]
\end{equation}

Taking the statistical uncertainty due to a finite number of samples into account, this analytic result coincides well with the ACF of the numerical evaluation shown above.

The ACF of a periodic signal is also periodic. The ACF $\varphi_{yy}[\kappa]$ is composed from a superposition of the ACF $\varphi_{xx}[\kappa]$ of the periodic cosine signal and an aperiodic contribution from the additive noise. Hence, the period of the cosine can be estimated from the period of the ACF.

It can be concluded from the CCF that the additive noise is not correlated with the cosine signal.

### Example - Denoising by Repeated Averaging

Let's assume that we can have access to multiple observations of a noisy deterministic signal $x[k]$. The $i$-th observation $y_i[k]$ is then given by the following AWGN model

\begin{equation}
y_i[k] = x[k] + n_i[k]
\end{equation}

where $n_i[k]$ denotes the $i$-th sample function of a zero-mean white noise process with PSD $N_0$. It is assumed that the sample functions of the noise process are mutually independent. An estimate $\hat{x}[k]$ of the deterministic signal $x[k]$ is given by averaging over $N$ observations

\begin{equation}
\hat{x}[k] = \frac{1}{N} \sum_{i=0}^{N-1} (x[k] + n_i[k]) = x[k] + \underbrace{\frac{1}{N} \sum_{i=0}^{N-1} n_i[k]}_{n[k]}
\end{equation}

The estimate consists of a superposition of the true signal $x[k]$ and noise $n[k]$.

The [signal-to-noise ratio](https://en.wikipedia.org/wiki/Signal-to-noise_ratio) (SNR) of the estimate is defined as the power of the true signal $x[k]$ divided by the power of the additive noise. Following this definition, the average SNR is given as

\begin{equation}
\mathrm{SNR} = 10 \cdot \log_{10} \left( \frac{P_x}{\sigma_n^2} \right) \quad \text{ in dB}
\end{equation}

where $P_x$ denotes the average power of the signal $x[k]$ and $\sigma_n^2$ the variance of the noise $n[k]$. The SNR of the estimate $\hat{x}[k]$ is computed in the following to quantify the gain in SNR achieved by averaging. The average power $P_x$ of the signal is assumed to be known, the power of the noise $\sigma_x^2$ can be computed from its ACF. Repeated application of above findings for the AWGN model yields

\begin{equation}
\sigma_n^2 = \frac{1}{N^2} \, ( N \cdot N_0 ) = \frac{N_0}{N}
\end{equation}

since the scaling $\frac{1}{N}$ in above sum results in a $\frac{1}{N^2}$ scaling of the variance and $N$ uncorrelated instances of $n_i[k]$ with power $N_0$ are summed up. The SNR of the estimate follows then as

\begin{equation}
\mathrm{SNR} = 10 \cdot \log_{10} \left( P_x \right) - 10 \cdot \log_{10} \left( N_0 \right) + 10 \cdot \log_{10} \left( N \right) \quad \text{ in dB}
\end{equation}

By averaging over $N$ observations, an average gain of $10 \cdot \log_{10} \left( N \right)$ dB in terms of SNR can be achieved.

The following numerical example illustrates the denoising of a cosine signal superimposed by unit variance white Gaussian noise. One particular observation $y_i[k]$, the estimate $\hat{x}[k]$ by averaging over $N$ observations and the resulting SNR is shown.

```python
K = 1024  # number of samples
Nmax = 50  # number of observations

# generate signals
x = np.cos(2*np.pi/50 * np.arange(N))
np.random.seed(2)
n = np.random.normal(size=(Nmax, K))
# AWGN model
y = np.tile(x, (Nmax,1)) + n
# repeated averaging up to Nmax
xhat = np.zeros_like(y)
for i in range(Nmax):
    xhat[i, :] = 1/(i+1) * np.sum(y[:i+1, :], axis=0)
# compute SNR for all averages
Px = np.var(x)
Pn = np.var(xhat - x, axis=1)
SNR = 10*np.log10(Px/Pn)

# plot results
plt.figure(figsize=(10, 10))

plt.subplot(311)
plt.stem(y[0, :100], use_line_collection=True)
plt.title('One observation')
plt.xlabel(r'$k$')
plt.ylabel(r'$y_0[k]$')
plt.ylim([-3, 3])

plt.subplot(312)
plt.stem(xhat[Nmax-1, :100], use_line_collection=True)
plt.title('Average over {:2.0f} observations'.format(Nmax))
plt.xlabel(r'$k$')
plt.ylabel(r'$\hat{x}[k]$')
plt.ylim([-3, 3])

plt.subplot(313)
plt.plot(range(1, Nmax+1), 10*np.log10(Px * range(1, Nmax+1)), '--', label='theory')
plt.plot(range(1, Nmax+1), SNR, label='simulated')
plt.title('SNR')
plt.xlabel('number of averages $N$')
plt.ylabel('SNR in dB')
plt.grid()
plt.legend()

plt.tight_layout()
```

**Excercise**

* How does the SNR change if the instances $n_i[k]$ of the additive noise are mutually correlated?

Solution: The gain in terms of SNR will be lower since the mutually correlated AWGNs do not average out in the estimate $\hat{x}[k]$. For instance, the averaging will have no effect if the samples of the additive noise are equal for all observed signals $y_i[k] = x[k] + n_0[k]$.

**Copyright**

This notebook is provided as [Open Educational Resource](https://en.wikipedia.org/wiki/Open_educational_resources). Feel free to use the notebook for your own purposes. The text is licensed under [Creative Commons Attribution 4.0](https://creativecommons.org/licenses/by/4.0/), the code of the IPython examples under the [MIT license](https://opensource.org/licenses/MIT). Please attribute the work as follows: *Sascha Spors, Digital Signal Processing - Lecture notes featuring computational examples*.


# Decaimiento radiactivo

Existen muchos modelos de decaimiento radiactivo, sin embargo, uno de los más sencillos es el considerar que la cantidad de material radioactivo decae de forma proporcional a la cantidad que tenga en un tiempo $t$. esto puede ser escrito de forma sencilla en el siguiente modelo:
$$\frac{\Delta N(t)}{\Delta t}\rightarrow\frac{dN}{dt}=\lambda N$$
Esta ecuación resulta de un promedio sobre un conjunto de atomos (Ensamble), esto se tiene para el caso en el que el numero de partículas $N\to\infty$ y el intervalo de observación $\Delta t \to 0$. La ecuación del modelo puede ser integrada y esta da como resultado algo llamado la ley expoenecial de decaimiento de los núcleos.

El siguiente programa efectua una simulación de del decaimiento de un nucleo.

Tomado de del libro A Survey of Computational Physics Introductory Computational Science de Landau, Paez, Bordeianu (Python Multimodal eTextBook Beta4.0)

```python
import numpy as np
import matplotlib.pylab as plt
import random
```

```python
Lambda = 0.001 # Decay c o n s t a n t
maximum = 200. ; timemax = 500 # Pa rams
number = nloop = maximum # I n i t i a l v a l u e
lambda1 = 0.001
maximum = 200. ; timemax = 500
number = nloop = maximum
for time in range(0 , timemax + 1 ) : # Time l o o p
    for atom in range(1 , int(number) + 1 ) : # Decay l o o p
        decay = random.random( )
        if(decay < Lambda):
            nloop +=  -1
        number = nloop
```

El programa anterior simula el numero de atomos en un nucleo en función del tiempo a partir de la generación de números aleatorios con distribución uniforme.
# Ejercicio 1
Modifique el codigo anterior para guardar los valores que se obtienen de tiempo y numero de atomos en el nucleo, prepare una gráfica en donde muestre cual es el comportamiento que tiene el número de atomos en función del tiempo.

```python
Lambda = 0.001 # Decay c o n s t a n t
maximum = 200. ; timemax = 500 # Pa rams
number = nloop = maximum # I n i t i a l v a l u e
lambda1 = 0.001
maximum = 200. ; timemax = 500
number = nloop = maximum
Atomos=[]
tiempo=[]
for time in range(0 , timemax + 1 ) : # Time l o o p
    for atom in range(1 , int(number) + 1 ) : # Decay l o o p
        decay = random.random( )
        if(decay < Lambda):
            nloop +=  -1
        number = nloop
    Atomos.append(number)
    tiempo.append(time)
plt.plot(tiempo,Atomos)
plt.show()
```

# Ejercicio 2
Modifique el codigo efectuado en el ejercicio 1 con el fin de que no sólo se efectue el comportamiento de 1 nucleo si no que se efectue un ensamble de 1000 posibles comportamientos del número de atomos en el nucleo.
Efectue un promedio de todos los valores obtenidos para cada trayectoria y prepare una gráfica con el comportamiento del promedio.

```python
num_experimentos=100
Avg=np.zeros(timemax+1)
for experimento in range(num_experimentos):
    Lambda = 0.001 # Decay c o n s t a n t
    maximum = 200. ; timemax = 500 # Pa rams
    number = nloop = maximum # I n i t i a l v a l u e
    for time in range(0 , timemax + 1 ) : # Time l o o p
        for atom in range(1 , int(number) + 1 ) : # Decay l o o p
            decay = random.random( )
            if(decay < Lambda):
                nloop +=  -1
            number = nloop
        Avg[time] += number/num_experimentos
plt.plot(range(0 , timemax + 1 ),Avg)
plt.show()
```

# Ejercicio 3
Tome una trayectoria de su preferencia y tome el logaritmo  del numero de atomos en el nucleo y grafique estos resultados, intente recrear la figura 5.5 del libro guia.

```python
plt.plot(tiempo,np.log(Atomos))
plt.show()
```

# Ejercicio 4

Genere una gráfica en donde muestre que para distintos valores de $N(0)$ en el tiempo inicial, el comportamiento de $\ln(N(t)$ vs $t$ es invariante ante la escogencia de $N(0)$

```python
for i in (100,200,300,500,1000):
    Lambda = 0.001 # Decay c o n s t a n t
    maximum = float(i)
    timemax= 500 # Pa rams
    number = nloop = maximum # I n i t i a l v a l u e
    Atomos=[]
    tiempo=[]
    for time in range(0 , timemax + 1 ) : # Time l o o p
        for atom in range(1 , int(number) + 1 ) : # Decay l o o p
            decay = random.random( )
            if(decay < Lambda):
                nloop +=  -1
            number = nloop
        Atomos.append(number)
        tiempo.append(time)
    plt.plot(tiempo,np.log(Atomos))
plt.show()
```

# Ejercicio 5

La función de autocorrelación se define como la correlación cruzada de la señal consigo misma.
Esta función resulta de gran utilidad para encontrar patrones repetitivos dentro de una señal, como la periodicidad de una señal enmascarada bajo el ruido o para identificar la frecuencia fundamental de una señal que no contiene dicha componente, pero aparecen numerosas frecuencias armónicas de esta.

Esta función está dada por:

\begin{equation}
\varphi(i)=\frac{1}{N}\sum_{j=1}^{N} \left(x_j - \langle X\rangle\right)\left(x_{i+j}-\langle X\rangle\right) \tag{D2}
\end{equation}

Y tiene la propiedad de que si se cumple:
\begin{equation}
\varphi(i=0)=\frac{1}{N}\sum_{j=1}^{N} \left(x_j - \langle X\rangle\right)^2=\langle x_j - \langle X\rangle\rangle^2=\sigma^2\tag{D3}
\end{equation}

\begin{equation}
\varphi(i\ne 0)= \langle x_j - \langle X\rangle\rangle\langle x_{i\ne j} - \langle X\rangle\rangle=0\hspace{5mm}
(\rightarrow{\rm White\ noise})\hspace{-12mm}
\tag{D4}
\end{equation}

se denomina que el proceso estocastico es un ruido blanco o que no tiene correlación alguna más que en el momento en el que este fue generado.

Haciendo uso de la función de ``numpy`` "correlate" calcule la función de correlación para una secuencia de puntos generados a partir de una distribución uniforme.

```python
x=np.random.uniform(0,3,1000)
corr=np.correlate(x-np.mean(x),x-np.mean(x),mode="full")
corr=corr/len(x)
```

```python
plt.plot(range(0,len(corr)//2 +1),corr[len(corr)//2:])
```

 Como se observa el valor del pico es de

```python
np.max(corr)
```

    0.7710989675733794

El cual coincide con la varianza de los numeros aleatorios

```python
np.var(x)
```

    0.7710989675733794

Y su promedio es:

```python
np.mean(corr)
```

    -2.7769460345801814e-19

Con lo cual sepuede ver que lo que se tiene acá es un ruido blanco generado a partir de una distribución uniforme

```python

```

```python

```

```python

```

```python

```


# Introduction to the Harmonic Oscillator

*Note:* Much of this is adapted/copied from https://flothesof.github.io/harmonic-oscillator-three-methods-solution.html

This week week we are going to begin studying molecular dynamics, which uses classical mechanics to study molecular systems. Our "hydrogen atom" in this section will be the 1D harmomic oscillator.

The harmonic oscillator is a system that, when displaced from its equilibrium position, experiences a restoring force F proportional to the displacement x:

$$F=-kx$$

The potential energy of this system is

$$V = {1 \over 2}k{x^2}$$

These are sometime rewritten as

$$ F=- \omega_0^2 m x, \text{ }  V(x) = {1 \over 2}  \omega_0^2 m {x^2}$$

Where $\omega_0  = \sqrt {{k \over m}} $

In classical mechanics, our goal is to determine the equations of motion, $x(t),y(t)$, that describe our system.

In this notebook we will use sympy to solve an second order, ordinary differential equation.

## 1. Solving differential equations with sympy

Soliving differential equations can be tough, and there is not always a set plan on how to proceed. Luckily for us, the harmonic osscillator is the classic second order diffferential eqations.

Consider the following second order differential equation

$$ay(t)''+by(t)'=c$$

where $y(t)'' = {{{d^2}y} \over {dt^2}}$, and $y(t)' = {{{d}y} \over {dt}}$

We can rewrite this as a homogeneous linear differential equations

$$ay(t)''+by(t)'-c=0$$

The goal here is to find $y(t)$, similar to our classical mechanics problems. Lets use sympy to solve this equation

### Second order ordinary differential equation

First we import the sympy library

```python
import sympy as sym
```

Next we initialize pretty printing

```python
sym.init_printing()
```

Next we will set our symbols

```python
t,a,b,c=sym.symbols("t,a,b,c")
```

Now for somehting new. We can define functions using `sym.Function("f")`

```python
y=sym.Function("y")
y(t)
```

Now, If I want to define a first or second derivative, I can use `sym.diff`

```python
sym.diff(y(t),(t,1)),sym.diff(y(t),(t,2))
```

My differential equation can be written as follows

```python
dfeq=a*sym.diff(y(t),(t,2))+b*sym.diff(y(t),(t,1))-c
dfeq
```

```python
sol = sym.dsolve(dfeq)
sol
```

The two constants $C_1$ and $C_2$ can be determined by setting boundry conditions.
First, we can set the condition $y(t=0)=y_0$

The next intial condition we will set is $y'(t=0)=v_0$

To setup the equality we want to solve, we are using `sym.Eq`. This function sets up an equaility between a lhs aand rhs of an equation

```python
# sym.Eq example
alpha,beta=sym.symbols("alpha,beta")
sym.Eq(alpha+2,beta)
```

Back to the actual problem

```python
y0,v0=sym.symbols("y_0,v_0")
ics=[sym.Eq(sol.args[1].subs(t, 0), y0),
     sym.Eq(sol.args[1].diff(t).subs(t, 0), v0)]
ics
```

We can use this result to first solve for $C_2$ and then solve for $C_1$.
Or we can use sympy to solve this for us.

```python
solved_ics=sym.solve(ics)
solved_ics
```

Substitute the result back into $y(t)$

```python
full_sol = sol.subs(solved_ics[0])
full_sol
```

We can plot this result too. Assume that $a,b,c=1$ and that the starting conditions are $y_0=0,v_0=0$

We will use two sample problems:

* case 1 : initial position is nonzero and initial velocity is zero
* case 2 : initial position is zero and initialvelocity is nonzero

```python
# Print plots
%matplotlib inline
```

#### Initial velocity set to zero

```python
case1 = sym.simplify(full_sol.subs({y0:0, v0:0, a:1, b:1, c:1}))
case1
```

```python
sym.plot(case1.rhs)
sym.plot(case1.rhs,(t,-2,2))
```

#### Initial velocity set to one

```python
case2 = sym.simplify(full_sol.subs({y0:0, v0:1, a:1, b:1, c:1}))
case2
```

```python
sym.plot(case2.lhs,(t,-2,2))
```

## Calculate the phase space

As we will see in lecture, the state of our classical systems are defined as points in phase space, a hyperspace defined by ${{\bf{r}}^N},{{\bf{p}}^N}$. We will convert our sympy expression into a numerical function so that we can plot the path of $y(t)$ in phase space $y,y'$.

```python
case1
```

```python
# Import numpy library
import numpy as np

# Make numerical functions out of symbolic expressions
yfunc=sym.lambdify(t,case1.rhs,'numpy')
vfunc=sym.lambdify(t,case1.rhs.diff(t),'numpy')

# Make list of numbers
tlst=np.linspace(-2,2,100)

# Import pyplot
import matplotlib
import matplotlib.pyplot as plt
# Make plot
plt.plot(yfunc(tlst),vfunc(tlst))
plt.xlabel('$y$')
plt.ylabel("$y'$")
plt.show()
```

### Exercise 1.1

Change the initial starting conditions and see how that changes the plots. Make three different plots with different starting conditions

```python
#Making initial velocity equal 10 and change initial positions to 5,5,5
case3 = sym.simplify(full_sol.subs({y0:1, v0:10, a:5, b:5, c:5}))
case3
```

```python
sym.plot(case3.rhs)
sym.plot(case3.rhs,(t,-2,2))
```

```python
case4 = sym.simplify(full_sol.subs({y0:5, v0:2, a:3, b:4, c:5}))
case4
```

```python
sym.plot(case4.rhs)
sym.plot(case4.rhs,(t,-2,2))
```

```python
case5 = sym.simplify(full_sol.subs({y0:10, v0:0, a:0, b:1, c:1}))
case5
```

```python
sym.plot(case5.rhs)
sym.plot(case5.rhs,(t,-2,2))
```

## 2. Harmonic oscillator

Applying the harmonic oscillator force to Newton's second law leads to the following second order differential equation

$$ F = m a $$

$$ F= - \omega_0^2 m x $$

$$ a = - \omega_0^2 x $$

$$ x(t)'' = - \omega_0^2 x $$

The final expression can be rearranged into a second order homogenous differential equation, and can be solved using the methods we used above

Your goal is determine and plot the equations of motion of a 1D harmomnic oscillator

### Exercise 2.1

1. Use the methodology above to determine the equations of motion $x(t), v(t)$ for a harmonic ocillator
1. Solve for any constants by using the following initial conditions: $x(0)=x_0, v(0)=v_0$
1. Show expressions for and plot the equations of motion for the following cases:
    1. $x(0)=0, v(0)=0$
    1. $x(0)=0, v(0)>0$
    1. $x(0)>0, v(0)=0$
    1. $x(0)<0, v(0)=0$
1. Plot the phasespace diagram for the harmonic oscillator

```python
# Your code here
m,t,omega=sym.symbols("m,t,omega")
x=sym.Function("x")
x(t)
```

```python
sym.diff(x(t),(t,1)),sym.diff(x(t),(t,2))
```

```python
dfeq1=sym.diff(x(t),(t,2))+omega**2*x(t)
dfeq1
```

```python
sol1 = sym.dsolve(dfeq1)
sol1
```

```python
x0,v0=sym.symbols("x_0,v_0")
ics1=[sym.Eq(sol1.args[1].subs(t, 0), x0),
     sym.Eq(sol1.args[1].diff(t).subs(t, 0), v0)]
ics1
```

```python
solved_ics1=sym.solve(ics1)
solved_ics1
```

```python
full_sol1 = sol.subs(solved_ics1[0])
full_sol1
```

```python
# Print plots
%matplotlib inline
```

```python
case_100 = sym.simplify(full_sol1.subs({x0:0, v0:0, omega:1}))
case_100
```

```python
sym.plot(case_100.rhs)
sym.plot(case_100.rhs,(t,-2,2))
```

```python
case_101 = sym.simplify(full_sol1.subs({x0:0, v0:5, omega:1}))
case_101
```

```python
sym.plot(case_101.rhs)
sym.plot(case_101.rhs,(t,-2,2))
```

```python
case_102 = sym.simplify(full_sol1.subs({x0:5, v0:0, omega:1}))
case_102
```

```python
sym.plot(case_102.rhs)
sym.plot(case_102.rhs,(t,-2,2))
```

```python
case_103 = sym.simplify(full_sol1.subs({x0:-5, v0:0, omega:1}))
case_103
```

```python
sym.plot(case_103.rhs)
sym.plot(case_103.rhs,(t,-2,2))
```

```python
case_100
```

```python
# Import numpy library
import numpy as np

# Make numerical functions out of symbolic expressions
xfunc=sym.lambdify(t,case_100.rhs,'numpy')
vfunc=sym.lambdify(t,case_100.rhs.diff(t),'numpy')

# Make list of numbers
tlst=np.linspace(-2,2,100)

# Import pyplot
import matplotlib
import matplotlib.pyplot as plt
# Make plot
plt.plot(xfunc(tlst),vfunc(tlst))
plt.xlabel('$y$')
plt.ylabel("$y'$")
plt.show()
```

```python
# Import numpy library
import numpy as np

# Make numerical functions out of symbolic expressions
xfunc=sym.lambdify(t,case_101.rhs,'numpy')
vfunc=sym.lambdify(t,case_101.rhs.diff(t),'numpy')

# Make list of numbers
tlst=np.linspace(-5,5,100)

# Import pyplot
import matplotlib
import matplotlib.pyplot as plt
# Make plot
plt.plot(xfunc(tlst),vfunc(tlst))
plt.xlabel('$y$')
plt.ylabel("$y'$")
plt.show()
```

```python
# Import numpy library
import numpy as np

# Make numerical functions out of symbolic expressions
xfunc=sym.lambdify(t,case_102.rhs,'numpy')
vfunc=sym.lambdify(t,case_102.rhs.diff(t),'numpy')

# Make list of numbers
tlst=np.linspace(-5,5,100)

# Import pyplot
import matplotlib
import matplotlib.pyplot as plt
# Make plot
plt.plot(xfunc(tlst),vfunc(tlst))
plt.xlabel('$y$')
plt.ylabel("$y'$")
plt.show()
```

```python

```


# Investigating the Re-use of subsamples from previous iterations

Context: a multi-fidelity optimization procedure where an Error Grid is created after every evaluation to determine the next best fidelity for evaluation. Each Error Grid is made up of 'pixels' $(n_h, n_l)$ where $n_h < N_H$ and $n_l < N_L$, where $N_H, N_L$ are the total number of high- and low-fidelity evaluations available and $n_h, n_l$ are the size of the subsample from that total.

The idea is as follows: In each iteration, a single new evaluation is added, i.e., $N_H += 1$ or $N_L += 1$. For each previously calculated Error Grid 'pixel' $(n_h, n_l)$, all subsamples used in that previous iteration are still valid subsamples: they simply happen to exclude the newly added point. Reusing these is therefore no problem in principle, but we want to avoid a bias from ignoring the new point (otherwise, what's the point in evaluating that in the first place if we don't bother learning from it). So the question is: if we were to uniformly at random draw a new set of subsamples based on the new $N_H$ or $N_L$, what percentage do we expect to **not** include the newest point? Assuming the previous set of subsamples is also drawn uniformly at random, a random subselection from those should suffice to be reused as that percentage for the new iteration. The remainder can then be drawn anew, with the explicit requirement that they should contain the new sample.

This procedure allows us to reuse some of the work from the previous iteration for all previously calculated pixels $(n_h, n_l)$, without introducing a bias towards including/excluding the newest sample.

(It must however be noted that this procedure is more 'exact' rather than an actual random distribution. While I don't expect this to cause a problem, this will have to be kept in mind in case it becomes relevant elsewhere in the process)

## Deriving formulas: how to calculate how much re-use is possible?

### Definitions/assumptions:

All high-fidelity samples are also evaluated in low-fidelity.

A subsample of size $(n_h, n_l)$ is created in two steps:
1. First a subsample of $n_h$ high-fidelity samples is drawn from the total number $N_H$. This means that the number of 'high-fidelity'-only subsamples is $N_H \choose n_h$
2. All high-fidelity samples are automatically included in the low-fidelity sample, meaning that only $n_l - n_h$ low-fidelity samples are left to be sampled from the $N_L - n_H$ remaining samples. This means the number of possible 'low-fidelity'-only subsamples after having drawn a 'high-fidelity'-only subsample is ${N_L - n_h} \choose {n_l - n_h}$

Since the drawing of the high- and low-fidelity subsamples are independent of eachother, the total number of subsamples for an Error Grid 'pixel' $n_h, n_l$ is ${N_H \choose n_h} \cdot {{N_L - n_h} \choose {n_l - n_h}}$

### Derivation of formulas

```python
import sympy
sympy.init_printing()

NH, NL, nh, nl = sympy.symbols('N_H N_L n_h n_l')
```

#### For $N_L += 1$

```python
num_subsamples = sympy.binomial(NH, nh) * sympy.binomial(NL-nh, nl-nh)
more_subsamples = sympy.binomial(NH, nh) * sympy.binomial((NL+1)-nh, nl-nh)

ratio = num_subsamples / more_subsamples
ratio  # binom(NH, nh) cancels out
```

```python
ratio.rewrite(sympy.factorial)
```

```python
sympy.simplify(ratio.rewrite(sympy.factorial))  # simplify: (x+1)! / x! = x+1  and x! / (x+1)! = 1/(x+1)
```

#### For $N_H += 1$

```python
num_subsamples = sympy.binomial(NH, nh) * sympy.binomial(NL-nh, nl-nh)
more_subsamples = sympy.binomial((NH+1), nh) * sympy.binomial(NL-nh, nl-nh)

ratio = num_subsamples / more_subsamples
ratio  # binom(NL-nh, nl-nh) cancels out
```

```python
ratio.rewrite(sympy.factorial)
```

```python
sympy.simplify(ratio.rewrite(sympy.factorial))  # simplify: (x+1)! / x! = x+1  and x! / (x+1)! = 1/(x+1)
```

#### Summary:

As can be seen in the derivations above, when adding a new low-fidelity sample, the fraction that can be reused is fairly simply defined by $\dfrac{N_L - n_l + 1}{N_L - n_h + 1}$, depending only on $N_L, n_h$ and $n_l$.

For the case of adding a high-fidelity sample, this is defined only as a relation between $N_H$ and $n_h$ by $\dfrac{N_H - n_h + 1}{N_H + 1}$

## Visual inspection of actual percentage reuse

```python
import matplotlib.pyplot as plt
import numpy as np
```

```python
def reuse_after_low_add(nh, nl, NL):
    """calculate the fraction of subsamples that can be reused
    if the number of low-fidelity samples is increased by 1"""
    return (NL-nl+1)/(NL-nh+1)

def reuse_after_high_add(nh, NH):
    """calculate the fraction of subsamples that can be reused
    if the number of high-fidelity samples is increased by 1"""
    return (NH - nh + 1) / (NH + 1)
```

```python
nh, NH = np.meshgrid(np.arange(2, 100), np.arange(3, 101))

ratio = reuse_after_high_add(nh, NH)
ratio[nh >= NH] = np.nan

fig = plt.figure()
ax = fig.gca()

img = ax.imshow(ratio, origin='lower')
ax.set_title('Ratio of reusable subsamples for $N_H$ += 1')
ax.set_xlabel('$n_h$')
ax.set_ylabel('$N_H$')
fig.colorbar(img)
fig.show()
```

```python
from ipywidgets import interact, IntSlider
%matplotlib inline

def update(nh):
    nl, NL = np.meshgrid(np.arange(2, 250, dtype=float), np.arange(3, 251, dtype=float))
    nl[(nl <= nh) | (nl >= NL)] = np.nan
    NL[(nl <= nh) | (nl >= NL)] = np.nan

    ratio = reuse_after_low_add(nh, nl, NL)

    fig = plt.figure()
    ax = fig.gca()

    img = ax.imshow(ratio, origin='lower')
    ax.set_title(f'Ratio of reusable subsamples for $N_L$ += 1 ($n_h$ = {nh})')
    ax.set_xlabel('$n_l$')
    ax.set_ylabel('$N_L$')
    fig.colorbar(img)
    fig.show()

interact(update, nh=IntSlider(value=10, min=1, max=200, step=1))
```

    interactive(children=(IntSlider(value=10, description='nh', max=200, min=1), Output()), _dom_classes=('widget-…

    <function __main__.update(nh)>

```python

```