```python
from sympy.physics.mechanics import *
import sympy as sp
mechanics_printing(pretty_print=True)
```

```python
m1, m2, m3, m4, l1, l2, l3, l4 = sp.symbols(r'm_1 m_2 m_3 m_4 l_1 l_2 l_3 l_4')
t, g, h = sp.symbols('t g h')
v1, v2, v3, v4 = dynamicsymbols(r'\theta_1 \theta_2 \theta_3 \theta_4')
dv1, dv2, dv3, dv4 = dynamicsymbols(r'\theta_1 \theta_2 \theta_3 \theta_4', 1)
ddv1, ddv2, ddv3, ddv4 = dynamicsymbols(r'\theta_1 \theta_2 \theta_3 \theta_4', 2)
```

```python
x1 = l1 * sp.sin(v1)
y1 = -l1 * sp.cos(v1)
x2 = x1 + l2 * sp.sin(v2)
y2 = y1 + -l2 * sp.cos(v2)
x3 = x2 + l3 * sp.sin(v3)
y3 = y2 + -l3 * sp.cos(v3)
x4 = x3 + l4 * sp.sin(v4)
y4 = y3 + -l4 * sp.cos(v4)

dx1 = x1.diff(t)
dy1 = y1.diff(t)
dx2 = x2.diff(t)
dy2 = y2.diff(t)
dx3 = x3.diff(t)
dy3 = y3.diff(t)
dx4 = x4.diff(t)
dy4 = y4.diff(t)
```

```python
V = (m1 * g * y1) + (m2 * g * y2) + (m3 * g * y3) + (m4 * g * y4)
T = (sp.Rational(1, 2) * m1 * (dx1**2 + dy1**2)) + (sp.Rational(1, 2) * m2 * (dx2**2+dy2**2)) + (sp.Rational(1, 2) * m3 * (dx3**2 + dy3**2)) + (sp.Rational(1, 2) * m4 * (dx4**2 + dy4**2))
L = T - V
```

```python
LM = LagrangesMethod(L, [v1, v2, v3, v4])
```

```python
soln = LM.form_lagranges_equations()
```

```python
sp.latex(soln)
```

    '\\left[\\begin{matrix}g l_{1} m_{1} \\sin{\\left(\\theta_{1}{\\left(t \\right)} \\right)} + g l_{1} m_{2} \\sin{\\left(\\theta_{1}{\\left(t \\right)} \\right)} + g l_{1} m_{3} \\sin{\\left(\\theta_{1}{\\left(t \\right)} \\right)} + g l_{1} m_{4} \\sin{\\left(\\theta_{1}{\\left(t \\right)} \\right)} + \\frac{m_{1} \\left(2 l_{1}^{2} \\sin^{2}{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{1}{\\left(t \\right)} + 2 l_{1}^{2} \\cos^{2}{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{1}{\\left(t \\right)}\\right)}{2} - \\frac{m_{2} \\left(2 l_{1} \\left(l_{1} \\sin{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{1}{\\left(t \\right)} + l_{2} \\sin{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{2}{\\left(t \\right)}\\right) \\cos{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{1}{\\left(t \\right)} - 2 l_{1} \\left(l_{1} \\cos{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{1}{\\left(t \\right)} + l_{2} \\cos{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{2}{\\left(t \\right)}\\right) \\sin{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{1}{\\left(t \\right)}\\right)}{2} + \\frac{m_{2} \\left(2 l_{1} \\left(l_{1} \\sin{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{1}{\\left(t \\right)} + l_{2} \\sin{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{2}{\\left(t \\right)}\\right) \\cos{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{1}{\\left(t \\right)} - 2 l_{1} \\left(l_{1} \\cos{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{1}{\\left(t \\right)} + l_{2} \\cos{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{2}{\\left(t \\right)}\\right) \\sin{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{1}{\\left(t \\right)} + 2 l_{1} \\left(- l_{1} \\sin{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{1}{\\left(t \\right)}\\right)^{2} + l_{1} \\cos{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{1}{\\left(t \\right)} - l_{2} \\sin{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{2}{\\left(t \\right)}\\right)^{2} + l_{2} \\cos{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{2}{\\left(t \\right)}\\right) \\cos{\\left(\\theta_{1}{\\left(t \\right)} \\right)} + 2 l_{1} \\left(l_{1} \\sin{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{1}{\\left(t \\right)} + l_{1} \\cos{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{1}{\\left(t \\right)}\\right)^{2} + l_{2} \\sin{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{2}{\\left(t \\right)} + l_{2} \\cos{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{2}{\\left(t \\right)}\\right)^{2}\\right) \\sin{\\left(\\theta_{1}{\\left(t \\right)} \\right)}\\right)}{2} - \\frac{m_{3} \\left(2 l_{1} \\left(l_{1} \\sin{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{1}{\\left(t \\right)} + l_{2} \\sin{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{2}{\\left(t \\right)} + l_{3} \\sin{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{3}{\\left(t \\right)}\\right) \\cos{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{1}{\\left(t \\right)} - 2 l_{1} \\left(l_{1} \\cos{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{1}{\\left(t \\right)} + l_{2} \\cos{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{2}{\\left(t \\right)} + l_{3} \\cos{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{3}{\\left(t \\right)}\\right) \\sin{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{1}{\\left(t \\right)}\\right)}{2} + \\frac{m_{3} \\left(2 l_{1} \\left(l_{1} \\sin{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{1}{\\left(t \\right)} + l_{2} \\sin{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{2}{\\left(t \\right)} + l_{3} \\sin{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{3}{\\left(t \\right)}\\right) \\cos{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{1}{\\left(t \\right)} - 2 l_{1} \\left(l_{1} \\cos{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{1}{\\left(t \\right)} + l_{2} \\cos{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{2}{\\left(t \\right)} + l_{3} \\cos{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{3}{\\left(t \\right)}\\right) \\sin{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{1}{\\left(t \\right)} + 2 l_{1} \\left(- l_{1} \\sin{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{1}{\\left(t \\right)}\\right)^{2} + l_{1} \\cos{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{1}{\\left(t \\right)} - l_{2} \\sin{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{2}{\\left(t \\right)}\\right)^{2} + l_{2} \\cos{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{2}{\\left(t \\right)} - l_{3} \\sin{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{3}{\\left(t \\right)}\\right)^{2} + l_{3} \\cos{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{3}{\\left(t \\right)}\\right) \\cos{\\left(\\theta_{1}{\\left(t \\right)} \\right)} + 2 l_{1} \\left(l_{1} \\sin{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{1}{\\left(t \\right)} + l_{1} \\cos{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{1}{\\left(t \\right)}\\right)^{2} + l_{2} \\sin{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{2}{\\left(t \\right)} + l_{2} \\cos{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{2}{\\left(t \\right)}\\right)^{2} + l_{3} \\sin{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{3}{\\left(t \\right)} + l_{3} \\cos{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{3}{\\left(t \\right)}\\right)^{2}\\right) \\sin{\\left(\\theta_{1}{\\left(t \\right)} \\right)}\\right)}{2} - \\frac{m_{4} \\left(2 l_{1} \\left(l_{1} \\sin{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{1}{\\left(t \\right)} + l_{2} \\sin{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{2}{\\left(t \\right)} + l_{3} \\sin{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{3}{\\left(t \\right)} + l_{4} \\sin{\\left(\\theta_{4}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{4}{\\left(t \\right)}\\right) \\cos{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{1}{\\left(t \\right)} - 2 l_{1} \\left(l_{1} \\cos{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{1}{\\left(t \\right)} + l_{2} \\cos{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{2}{\\left(t \\right)} + l_{3} \\cos{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{3}{\\left(t \\right)} + l_{4} \\cos{\\left(\\theta_{4}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{4}{\\left(t \\right)}\\right) \\sin{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{1}{\\left(t \\right)}\\right)}{2} + \\frac{m_{4} \\left(2 l_{1} \\left(l_{1} \\sin{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{1}{\\left(t \\right)} + l_{2} \\sin{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{2}{\\left(t \\right)} + l_{3} \\sin{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{3}{\\left(t \\right)} + l_{4} \\sin{\\left(\\theta_{4}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{4}{\\left(t \\right)}\\right) \\cos{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{1}{\\left(t \\right)} - 2 l_{1} \\left(l_{1} \\cos{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{1}{\\left(t \\right)} + l_{2} \\cos{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{2}{\\left(t \\right)} + l_{3} \\cos{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{3}{\\left(t \\right)} + l_{4} \\cos{\\left(\\theta_{4}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{4}{\\left(t \\right)}\\right) \\sin{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{1}{\\left(t \\right)} + 2 l_{1} \\left(- l_{1} \\sin{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{1}{\\left(t \\right)}\\right)^{2} + l_{1} \\cos{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{1}{\\left(t \\right)} - l_{2} \\sin{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{2}{\\left(t \\right)}\\right)^{2} + l_{2} \\cos{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{2}{\\left(t \\right)} - l_{3} \\sin{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{3}{\\left(t \\right)}\\right)^{2} + l_{3} \\cos{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{3}{\\left(t \\right)} - l_{4} \\sin{\\left(\\theta_{4}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{4}{\\left(t \\right)}\\right)^{2} + l_{4} \\cos{\\left(\\theta_{4}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{4}{\\left(t \\right)}\\right) \\cos{\\left(\\theta_{1}{\\left(t \\right)} \\right)} + 2 l_{1} \\left(l_{1} \\sin{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{1}{\\left(t \\right)} + l_{1} \\cos{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{1}{\\left(t \\right)}\\right)^{2} + l_{2} \\sin{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{2}{\\left(t \\right)} + l_{2} \\cos{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{2}{\\left(t \\right)}\\right)^{2} + l_{3} \\sin{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{3}{\\left(t \\right)} + l_{3} \\cos{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{3}{\\left(t \\right)}\\right)^{2} + l_{4} \\sin{\\left(\\theta_{4}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{4}{\\left(t \\right)} + l_{4} \\cos{\\left(\\theta_{4}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{4}{\\left(t \\right)}\\right)^{2}\\right) \\sin{\\left(\\theta_{1}{\\left(t \\right)} \\right)}\\right)}{2}\\\\g l_{2} m_{2} \\sin{\\left(\\theta_{2}{\\left(t \\right)} \\right)} + g l_{2} m_{3} \\sin{\\left(\\theta_{2}{\\left(t \\right)} \\right)} + g l_{2} m_{4} \\sin{\\left(\\theta_{2}{\\left(t \\right)} \\right)} - \\frac{m_{2} \\left(2 l_{2} \\left(l_{1} \\sin{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{1}{\\left(t \\right)} + l_{2} \\sin{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{2}{\\left(t \\right)}\\right) \\cos{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{2}{\\left(t \\right)} - 2 l_{2} \\left(l_{1} \\cos{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{1}{\\left(t \\right)} + l_{2} \\cos{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{2}{\\left(t \\right)}\\right) \\sin{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{2}{\\left(t \\right)}\\right)}{2} + \\frac{m_{2} \\left(2 l_{2} \\left(l_{1} \\sin{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{1}{\\left(t \\right)} + l_{2} \\sin{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{2}{\\left(t \\right)}\\right) \\cos{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{2}{\\left(t \\right)} - 2 l_{2} \\left(l_{1} \\cos{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{1}{\\left(t \\right)} + l_{2} \\cos{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{2}{\\left(t \\right)}\\right) \\sin{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{2}{\\left(t \\right)} + 2 l_{2} \\left(- l_{1} \\sin{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{1}{\\left(t \\right)}\\right)^{2} + l_{1} \\cos{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{1}{\\left(t \\right)} - l_{2} \\sin{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{2}{\\left(t \\right)}\\right)^{2} + l_{2} \\cos{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{2}{\\left(t \\right)}\\right) \\cos{\\left(\\theta_{2}{\\left(t \\right)} \\right)} + 2 l_{2} \\left(l_{1} \\sin{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{1}{\\left(t \\right)} + l_{1} \\cos{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{1}{\\left(t \\right)}\\right)^{2} + l_{2} \\sin{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{2}{\\left(t \\right)} + l_{2} \\cos{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{2}{\\left(t \\right)}\\right)^{2}\\right) \\sin{\\left(\\theta_{2}{\\left(t \\right)} \\right)}\\right)}{2} - \\frac{m_{3} \\left(2 l_{2} \\left(l_{1} \\sin{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{1}{\\left(t \\right)} + l_{2} \\sin{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{2}{\\left(t \\right)} + l_{3} \\sin{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{3}{\\left(t \\right)}\\right) \\cos{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{2}{\\left(t \\right)} - 2 l_{2} \\left(l_{1} \\cos{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{1}{\\left(t \\right)} + l_{2} \\cos{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{2}{\\left(t \\right)} + l_{3} \\cos{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{3}{\\left(t \\right)}\\right) \\sin{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{2}{\\left(t \\right)}\\right)}{2} + \\frac{m_{3} \\left(2 l_{2} \\left(l_{1} \\sin{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{1}{\\left(t \\right)} + l_{2} \\sin{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{2}{\\left(t \\right)} + l_{3} \\sin{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{3}{\\left(t \\right)}\\right) \\cos{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{2}{\\left(t \\right)} - 2 l_{2} \\left(l_{1} \\cos{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{1}{\\left(t \\right)} + l_{2} \\cos{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{2}{\\left(t \\right)} + l_{3} \\cos{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{3}{\\left(t \\right)}\\right) \\sin{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{2}{\\left(t \\right)} + 2 l_{2} \\left(- l_{1} \\sin{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{1}{\\left(t \\right)}\\right)^{2} + l_{1} \\cos{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{1}{\\left(t \\right)} - l_{2} \\sin{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{2}{\\left(t \\right)}\\right)^{2} + l_{2} \\cos{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{2}{\\left(t \\right)} - l_{3} \\sin{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{3}{\\left(t \\right)}\\right)^{2} + l_{3} \\cos{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{3}{\\left(t \\right)}\\right) \\cos{\\left(\\theta_{2}{\\left(t \\right)} \\right)} + 2 l_{2} \\left(l_{1} \\sin{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{1}{\\left(t \\right)} + l_{1} \\cos{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{1}{\\left(t \\right)}\\right)^{2} + l_{2} \\sin{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{2}{\\left(t \\right)} + l_{2} \\cos{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{2}{\\left(t \\right)}\\right)^{2} + l_{3} \\sin{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{3}{\\left(t \\right)} + l_{3} \\cos{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{3}{\\left(t \\right)}\\right)^{2}\\right) \\sin{\\left(\\theta_{2}{\\left(t \\right)} \\right)}\\right)}{2} - \\frac{m_{4} \\left(2 l_{2} \\left(l_{1} \\sin{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{1}{\\left(t \\right)} + l_{2} \\sin{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{2}{\\left(t \\right)} + l_{3} \\sin{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{3}{\\left(t \\right)} + l_{4} \\sin{\\left(\\theta_{4}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{4}{\\left(t \\right)}\\right) \\cos{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{2}{\\left(t \\right)} - 2 l_{2} \\left(l_{1} \\cos{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{1}{\\left(t \\right)} + l_{2} \\cos{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{2}{\\left(t \\right)} + l_{3} \\cos{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{3}{\\left(t \\right)} + l_{4} \\cos{\\left(\\theta_{4}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{4}{\\left(t \\right)}\\right) \\sin{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{2}{\\left(t \\right)}\\right)}{2} + \\frac{m_{4} \\left(2 l_{2} \\left(l_{1} \\sin{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{1}{\\left(t \\right)} + l_{2} \\sin{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{2}{\\left(t \\right)} + l_{3} \\sin{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{3}{\\left(t \\right)} + l_{4} \\sin{\\left(\\theta_{4}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{4}{\\left(t \\right)}\\right) \\cos{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{2}{\\left(t \\right)} - 2 l_{2} \\left(l_{1} \\cos{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{1}{\\left(t \\right)} + l_{2} \\cos{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{2}{\\left(t \\right)} + l_{3} \\cos{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{3}{\\left(t \\right)} + l_{4} \\cos{\\left(\\theta_{4}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{4}{\\left(t \\right)}\\right) \\sin{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{2}{\\left(t \\right)} + 2 l_{2} \\left(- l_{1} \\sin{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{1}{\\left(t \\right)}\\right)^{2} + l_{1} \\cos{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{1}{\\left(t \\right)} - l_{2} \\sin{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{2}{\\left(t \\right)}\\right)^{2} + l_{2} \\cos{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{2}{\\left(t \\right)} - l_{3} \\sin{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{3}{\\left(t \\right)}\\right)^{2} + l_{3} \\cos{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{3}{\\left(t \\right)} - l_{4} \\sin{\\left(\\theta_{4}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{4}{\\left(t \\right)}\\right)^{2} + l_{4} \\cos{\\left(\\theta_{4}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{4}{\\left(t \\right)}\\right) \\cos{\\left(\\theta_{2}{\\left(t \\right)} \\right)} + 2 l_{2} \\left(l_{1} \\sin{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{1}{\\left(t \\right)} + l_{1} \\cos{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{1}{\\left(t \\right)}\\right)^{2} + l_{2} \\sin{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{2}{\\left(t \\right)} + l_{2} \\cos{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{2}{\\left(t \\right)}\\right)^{2} + l_{3} \\sin{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{3}{\\left(t \\right)} + l_{3} \\cos{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{3}{\\left(t \\right)}\\right)^{2} + l_{4} \\sin{\\left(\\theta_{4}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{4}{\\left(t \\right)} + l_{4} \\cos{\\left(\\theta_{4}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{4}{\\left(t \\right)}\\right)^{2}\\right) \\sin{\\left(\\theta_{2}{\\left(t \\right)} \\right)}\\right)}{2}\\\\g l_{3} m_{3} \\sin{\\left(\\theta_{3}{\\left(t \\right)} \\right)} + g l_{3} m_{4} \\sin{\\left(\\theta_{3}{\\left(t \\right)} \\right)} - \\frac{m_{3} \\left(2 l_{3} \\left(l_{1} \\sin{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{1}{\\left(t \\right)} + l_{2} \\sin{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{2}{\\left(t \\right)} + l_{3} \\sin{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{3}{\\left(t \\right)}\\right) \\cos{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{3}{\\left(t \\right)} - 2 l_{3} \\left(l_{1} \\cos{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{1}{\\left(t \\right)} + l_{2} \\cos{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{2}{\\left(t \\right)} + l_{3} \\cos{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{3}{\\left(t \\right)}\\right) \\sin{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{3}{\\left(t \\right)}\\right)}{2} + \\frac{m_{3} \\left(2 l_{3} \\left(l_{1} \\sin{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{1}{\\left(t \\right)} + l_{2} \\sin{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{2}{\\left(t \\right)} + l_{3} \\sin{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{3}{\\left(t \\right)}\\right) \\cos{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{3}{\\left(t \\right)} - 2 l_{3} \\left(l_{1} \\cos{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{1}{\\left(t \\right)} + l_{2} \\cos{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{2}{\\left(t \\right)} + l_{3} \\cos{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{3}{\\left(t \\right)}\\right) \\sin{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{3}{\\left(t \\right)} + 2 l_{3} \\left(- l_{1} \\sin{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{1}{\\left(t \\right)}\\right)^{2} + l_{1} \\cos{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{1}{\\left(t \\right)} - l_{2} \\sin{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{2}{\\left(t \\right)}\\right)^{2} + l_{2} \\cos{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{2}{\\left(t \\right)} - l_{3} \\sin{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{3}{\\left(t \\right)}\\right)^{2} + l_{3} \\cos{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{3}{\\left(t \\right)}\\right) \\cos{\\left(\\theta_{3}{\\left(t \\right)} \\right)} + 2 l_{3} \\left(l_{1} \\sin{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{1}{\\left(t \\right)} + l_{1} \\cos{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{1}{\\left(t \\right)}\\right)^{2} + l_{2} \\sin{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{2}{\\left(t \\right)} + l_{2} \\cos{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{2}{\\left(t \\right)}\\right)^{2} + l_{3} \\sin{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{3}{\\left(t \\right)} + l_{3} \\cos{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{3}{\\left(t \\right)}\\right)^{2}\\right) \\sin{\\left(\\theta_{3}{\\left(t \\right)} \\right)}\\right)}{2} - \\frac{m_{4} \\left(2 l_{3} \\left(l_{1} \\sin{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{1}{\\left(t \\right)} + l_{2} \\sin{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{2}{\\left(t \\right)} + l_{3} \\sin{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{3}{\\left(t \\right)} + l_{4} \\sin{\\left(\\theta_{4}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{4}{\\left(t \\right)}\\right) \\cos{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{3}{\\left(t \\right)} - 2 l_{3} \\left(l_{1} \\cos{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{1}{\\left(t \\right)} + l_{2} \\cos{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{2}{\\left(t \\right)} + l_{3} \\cos{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{3}{\\left(t \\right)} + l_{4} \\cos{\\left(\\theta_{4}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{4}{\\left(t \\right)}\\right) \\sin{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{3}{\\left(t \\right)}\\right)}{2} + \\frac{m_{4} \\left(2 l_{3} \\left(l_{1} \\sin{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{1}{\\left(t \\right)} + l_{2} \\sin{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{2}{\\left(t \\right)} + l_{3} \\sin{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{3}{\\left(t \\right)} + l_{4} \\sin{\\left(\\theta_{4}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{4}{\\left(t \\right)}\\right) \\cos{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{3}{\\left(t \\right)} - 2 l_{3} \\left(l_{1} \\cos{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{1}{\\left(t \\right)} + l_{2} \\cos{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{2}{\\left(t \\right)} + l_{3} \\cos{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{3}{\\left(t \\right)} + l_{4} \\cos{\\left(\\theta_{4}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{4}{\\left(t \\right)}\\right) \\sin{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{3}{\\left(t \\right)} + 2 l_{3} \\left(- l_{1} \\sin{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{1}{\\left(t \\right)}\\right)^{2} + l_{1} \\cos{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{1}{\\left(t \\right)} - l_{2} \\sin{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{2}{\\left(t \\right)}\\right)^{2} + l_{2} \\cos{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{2}{\\left(t \\right)} - l_{3} \\sin{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{3}{\\left(t \\right)}\\right)^{2} + l_{3} \\cos{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{3}{\\left(t \\right)} - l_{4} \\sin{\\left(\\theta_{4}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{4}{\\left(t \\right)}\\right)^{2} + l_{4} \\cos{\\left(\\theta_{4}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{4}{\\left(t \\right)}\\right) \\cos{\\left(\\theta_{3}{\\left(t \\right)} \\right)} + 2 l_{3} \\left(l_{1} \\sin{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{1}{\\left(t \\right)} + l_{1} \\cos{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{1}{\\left(t \\right)}\\right)^{2} + l_{2} \\sin{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{2}{\\left(t \\right)} + l_{2} \\cos{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{2}{\\left(t \\right)}\\right)^{2} + l_{3} \\sin{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{3}{\\left(t \\right)} + l_{3} \\cos{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{3}{\\left(t \\right)}\\right)^{2} + l_{4} \\sin{\\left(\\theta_{4}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{4}{\\left(t \\right)} + l_{4} \\cos{\\left(\\theta_{4}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{4}{\\left(t \\right)}\\right)^{2}\\right) \\sin{\\left(\\theta_{3}{\\left(t \\right)} \\right)}\\right)}{2}\\\\g l_{4} m_{4} \\sin{\\left(\\theta_{4}{\\left(t \\right)} \\right)} - \\frac{m_{4} \\left(2 l_{4} \\left(l_{1} \\sin{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{1}{\\left(t \\right)} + l_{2} \\sin{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{2}{\\left(t \\right)} + l_{3} \\sin{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{3}{\\left(t \\right)} + l_{4} \\sin{\\left(\\theta_{4}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{4}{\\left(t \\right)}\\right) \\cos{\\left(\\theta_{4}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{4}{\\left(t \\right)} - 2 l_{4} \\left(l_{1} \\cos{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{1}{\\left(t \\right)} + l_{2} \\cos{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{2}{\\left(t \\right)} + l_{3} \\cos{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{3}{\\left(t \\right)} + l_{4} \\cos{\\left(\\theta_{4}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{4}{\\left(t \\right)}\\right) \\sin{\\left(\\theta_{4}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{4}{\\left(t \\right)}\\right)}{2} + \\frac{m_{4} \\left(2 l_{4} \\left(l_{1} \\sin{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{1}{\\left(t \\right)} + l_{2} \\sin{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{2}{\\left(t \\right)} + l_{3} \\sin{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{3}{\\left(t \\right)} + l_{4} \\sin{\\left(\\theta_{4}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{4}{\\left(t \\right)}\\right) \\cos{\\left(\\theta_{4}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{4}{\\left(t \\right)} - 2 l_{4} \\left(l_{1} \\cos{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{1}{\\left(t \\right)} + l_{2} \\cos{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{2}{\\left(t \\right)} + l_{3} \\cos{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{3}{\\left(t \\right)} + l_{4} \\cos{\\left(\\theta_{4}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{4}{\\left(t \\right)}\\right) \\sin{\\left(\\theta_{4}{\\left(t \\right)} \\right)} \\frac{d}{d t} \\theta_{4}{\\left(t \\right)} + 2 l_{4} \\left(- l_{1} \\sin{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{1}{\\left(t \\right)}\\right)^{2} + l_{1} \\cos{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{1}{\\left(t \\right)} - l_{2} \\sin{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{2}{\\left(t \\right)}\\right)^{2} + l_{2} \\cos{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{2}{\\left(t \\right)} - l_{3} \\sin{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{3}{\\left(t \\right)}\\right)^{2} + l_{3} \\cos{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{3}{\\left(t \\right)} - l_{4} \\sin{\\left(\\theta_{4}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{4}{\\left(t \\right)}\\right)^{2} + l_{4} \\cos{\\left(\\theta_{4}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{4}{\\left(t \\right)}\\right) \\cos{\\left(\\theta_{4}{\\left(t \\right)} \\right)} + 2 l_{4} \\left(l_{1} \\sin{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{1}{\\left(t \\right)} + l_{1} \\cos{\\left(\\theta_{1}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{1}{\\left(t \\right)}\\right)^{2} + l_{2} \\sin{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{2}{\\left(t \\right)} + l_{2} \\cos{\\left(\\theta_{2}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{2}{\\left(t \\right)}\\right)^{2} + l_{3} \\sin{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{3}{\\left(t \\right)} + l_{3} \\cos{\\left(\\theta_{3}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{3}{\\left(t \\right)}\\right)^{2} + l_{4} \\sin{\\left(\\theta_{4}{\\left(t \\right)} \\right)} \\frac{d^{2}}{d t^{2}} \\theta_{4}{\\left(t \\right)} + l_{4} \\cos{\\left(\\theta_{4}{\\left(t \\right)} \\right)} \\left(\\frac{d}{d t} \\theta_{4}{\\left(t \\right)}\\right)^{2}\\right) \\sin{\\left(\\theta_{4}{\\left(t \\right)} \\right)}\\right)}{2}\\end{matrix}\\right]'

```python
soln1 = sp.simplify(soln[0])
soln2 = sp.simplify(soln[1])
soln3 = sp.simplify(soln[2])
soln4 = sp.simplify(soln[3])
```

```python
solution = sp.solve([soln1, soln2, soln3, soln4], (ddv1, ddv2, ddv3, ddv4))
```

```python
sp.latex(solution)
```

```python

```


# Model Project - DYE code
# Overlapping Generations Model
## 1. Household optimization and Intertemporal budget constraint
We assume that the consumer has access to a competitive financial market, thus they are able to borrow or save at interest rate $r$. The utility of a houseehold over two periods (the household life-time utility) is given by the following time-additive utility function:

$$U=u(c_1)+\delta u(c_2),\quad u'>0,\quad u''<0,\quad 0<\delta\leq1$$

$c_1$ is consumption in period 1 and $c_2$ is the corresponding in period 2. $\delta\equiv\frac{1}{1-\rho}$ captures the preference between consumption in period 1 and 2. $\rho$ is called the *rate of time prefference* and captures the households impacience. Note that the utility of consumption in period 2 is discounted by $\delta$.

For this project, we will use the Constant-Relative Risk Aversion (CRRA) utility function of the form:

$$
\begin{equation}
u\left(c_{t}\right)=\begin{cases}
\frac{c^{1-\alpha}}{1-\alpha}, & \text{if } \alpha>0 \text{ and } \alpha \neq 1 \\
ln\left(c\right) & \text{if } \alpha=1
\end{cases}
\end{equation}
$$

We assume initial endowments in period 1 and 2, $e_1$ and $e_2$. One can think of $e_1$ as an income from period $t=0$ and $e_2$ as income from $t=1$. Given endowments in period 1, $e_1$ and period 2, $e_2$, households maximize their life-time utility, subject to their intertemporal budget constraint

$$\underset{c_1,c_2}{\text{max}}\quad u(c_1)+\delta u(c_2)$$

$$s.t.\quad c_1 + \frac{c_2}{1+r}  = e_1 + \frac{e_2}{1+r} $$

The first order condition is as follows:

$${c_1^*}^{-\alpha} = \delta (1+r) {c_2^*}^{-\alpha}$$

$$\Leftrightarrow c_2^* = \left [\delta (1+r) \right]^\frac{1}{\alpha}c_1^* $$

From the budget constraint we have:

$$c_2^* = E[e](1+r)-c_1^*(1+r),\quad E[e] \equiv e_1 + \frac{e_2}{1+r}$$

Solving for $c_1^*$ (from the FOC and this binding budget):

$$c_1^*  = \frac{E[e]}{1+\frac{\left [\delta (1+r) \right]^\frac{1}{\alpha}}{1+r}}$$

## 2. Defining model in python
We will now define the above model in python before continuing to our analysis. Defining the model beforehand, will ensure a clean structure and make the analysis easier to interpret, without too much code mixed into the analysis itself. We start off as usual, by importing the necessary packages for this analysis.

```python
# a. Import Packages
%matplotlib inline
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from ipywidgets import interact, fixed
```

Now, we define the model described as above.

```python
# b. Defining functions
def u(c, alpha):
    """ Defines the CRRA utility function (Constant-Relative Risk Aversion)
    """
    return (1/alpha)* c**(1-alpha)

def U(c1, c2, alpha, delta):
    """ Defines the households life-time utility function
    """
    return u(c1, alpha) + delta*u(c2, alpha)

def bud_cons(c1, r, e1, e2):
    """ Defines the budget constraint

        e = e1 + e2/(1+r) is total endowment
    """
    e = e1 + e2/(1+r)
    return e*(1+r) - c1*(1+r)

def indif_curve(c1, ubar, alpha, delta):
    """ Defines the indifference curve
    """
    return  ( ((1-alpha)/delta)*(ubar - u(c1, alpha)) )**(1/(1-alpha))
```

Define a function for solving the model:

```python
def solve_opt(r, alpha, delta, e1, e2):
    """ This function solves the model by defining the first order condition.
        Returning consumption in both periods as well as the life-time utility
    """
    e = e1 + e2/(1+r)
    A = (delta*(1+r))**(1/alpha)
    c1 = e/(1+A/(1+r))
    c2 = c1*A
    u = U(c1, c2, alpha, delta)
    return c1, c2, u
```

## 3. Preparing function for plotting
The last step before making the analysis, is to prepare a function, that can plot the model for later use.
We start by defining a set of pre-defined parameters and then continue to build a function.

```python
# c. Parameter definitions
alpha = 0.5
delta = 1
r = 0
e1, e2 = 80, 20

rmin, rmax = 0, 1
cmax = 150
```

```python
# d. Defining plotting function
def cons_plot(r, delta, alpha, e1, e2):
    """ This function will create a graph of the intertemporal utility and the budget constraint
    """
    c1 = np.linspace(0.1,cmax,num=100)
    c1e, c2e, uebar = solve_opt(r, alpha, delta, e1, e2)
    idfc = indif_curve(c1, uebar, alpha, delta)
    budg = bud_cons(c1,  r, e1, e2)

    fig, ax = plt.subplots(figsize=(8,8))
    ax.plot(c1, budg, lw=2.5)
    ax.plot(c1, idfc, lw=2.5)
    ax.vlines(c1e,0,c2e, linestyles="dashed")
    ax.hlines(c2e,0,c1e, linestyles="dashed")
    ax.plot(c1e,c2e,'ob')
    ax.vlines(e1,0,e2, linestyles="dashed")
    ax.hlines(e2,0,e1, linestyles="dashed")
    ax.plot(e1,e2,'ob')
    ax.text(e1-7,e2-6,r'$e^*$',fontsize=16)
    ax.set_xlim(0, cmax)
    ax.set_ylim(0, cmax)
    ax.set_xlabel(r'$c_1$', fontsize=16)
    ax.set_ylabel('$c_2$', fontsize=16)
    ax.spines['right'].set_visible(False)
    ax.spines['top'].set_visible(False)
    ax.fill_between(c1, bud_cons(c1, r, e1, e2), alpha = 0.2)
    ax.grid()
    plt.show()
```

## 4. Borrowing and saving
We will now use the functions that we have defined until now, to visualise two cases: Borrowing and saving. We will specify each case and then plot each case, in order to visualise.
### 4.1 Saving
If the consumer saves in period 1, we have the case where:   $s_1^* = e_1 - c_1^* >0$.

If we let $e_1=80$ and $c=50$, whilst assuming that the interest rate is zero and the rate of time prefference, $\delta=0$, thus ensuring that the consumer is indiferent between consuming in period 1 and period 2. In such case, consumption, endowment and savings is given as:

```python
# e. Define function that returns allocation as a pandas frame and graph
def allocation():
    c1e, c2e, uebar = solve_opt(r, alpha, delta, e1, e2)

    lst = {"Consumption": [c1e,c2e],
           "Endowment": [e1,e2],
           "Saving": [e1-c1e,0]}
    """ Creating a list with Consumption, Endowment and Saving as variables
        with their respective values
    """

    df = pd.DataFrame(lst,columns= ["Consumption", "Endowment", "Saving"])
    df.index = ["Period 1", "Period2"]
    """ The code above creates our Pandas Dataframe which takes lst as input and applies the variables
        as column names
    """

    return df
```

```python
# f.1 Allocation in the saving case
allocation()
```

      Consumption
      Endowment
      Saving

      Period 1
      50.0
      80
      30.0

      Period2
      50.0
      20
      0.0

This can be visualised as follows:

```python
cons_plot(r, delta, alpha, e1, e2)
```

### 4.1 Borrowing
If the consumer borrows in period 1, we have the case where:   $s_1^* = e_1 - c_1^* <0$.

If we let $e_1=50$ and $c=80$, whilse again assuming $r=0$ and $\delta=0$, then consumption, endowment and savings is given as:

```python
# f.2 Allocation in the borrowing case
e1,e2 = 20,80
allocation()
```

      Consumption
      Endowment
      Saving

      Period 1
      50.0
      20
      -30.0

      Period2
      50.0
      80
      0.0

```python
cons_plot(r, delta, alpha, e1, e2)
```

## 5. Conclusion: The effect of interest rate and time-preference on consumption
We will now turn our attention on how consumption allocation and utility is affected by $r$ and $\rho$.

From the expression: $c_2^* = \left [\delta (1+r) \right]^\frac{1}{\alpha}c_1^* $,  we get that an increase in $\rho$, causes a decrease in $\delta$. A decrease in $\delta$, will then cause a decrease in $c_2^*$, and conversely an increase in $c_1^*$. $\rho$ thus acts as a discount factor on utility, meaning an increse in rho causes consumer preference to change, such that the value consumption now, more than future consumption.

The interest rate, $r$, then has the opposite effect. An increase in $r$, will cause the consumer to save more, as this will increase $c_2$, due to the interest that will be received on savings. In other words, consumption in period one, becomes more expensive compared to consumption in period 2. This affects the budget constraint.

This means that the decision to save or borrow, depends on the parameters $r$ anf $\rho$. The intuition behind this is simple, if $\rho < r$, then the marginal benefit of consuming now, is less than the marginal benefit of saving for future consumption, thud the consumer will sacrifice consumption the first period, in order to consume more in second period and vice versa. We can summarize this as follows:
* If $\rho < r \Rightarrow u'(c_1) > u'(c_2)$ in other words, $c_1 < c_2$
* If $\rho > r \Rightarrow u'(c_1)  c_2$

The ineractive plot below, provides as a usefull tool to understanding this. The plot can show how changes in $r$ and $delta$ respectively, affects utillity and the budget constraint. Also, it shows that when $\rho < r$ then $c_1 < c_2$ and vice versa.

```python
interact(cons_plot, r=(rmin,rmax,0.1), alpha=fixed(alpha), delta=(0.5,1,0.1), e1=(10,100,1), e2=(10,100,1));
```

    interactive(children=(FloatSlider(value=0.0, description='r', max=1.0), FloatSlider(value=0.7, description='de…

```python

```

modelproject/Group DYE Code - Modelproject final.ipynb


# Logistic Regression

## From Regression to Classification

In the previous chapter we discussed linear regression and its application. It is a very valuable tool because

* it is easy to fit
* has simple interpretation and
* tests of statistical significance can be performed without difficulty.

But it has its drawbacks too. For example: it makes quite strong assumptions about the functional form of $f(X)$ and the error terms. Another disadvantage is that linear regression is not suited for classification problems where we deal with qualitative responses.

Below figure, where we regress the probability of default onto credit card balance, visualizes the issue.

* Linear regression produces negative probabilities for some balances
* Due to the large number of non-defaults, the linear regression's probability of default barely rises above 25%. This begs the question where we should set the threshold to qualify a default case. It can't be 50%, can it?

Logistic regression is able to overcome these obstacles. Rather than modeling the default class $y$ (e.g. 0 for non-default, 1 for default) it models the (conditional) probability that $y$ belongs to a certain category, $\Pr(Y=y|X)$.

## Logistic Model

Above figure estimated the probability of default given a client's credit card balance. In the same way we could have used a multiple linear regression to estimate the probability.

$$\begin{equation}
\Pr(Y=y|X) = \beta_0 + \beta_1 X_1 + \ldots + \beta_p X_p
\end{equation}$$

But as discussed above for the simple case, this too would yield unbounded probabilities (For convenience let us assume that we are using the generic 0/1 coding for the dependent variable). with $y > 1$ and $y<0$ for some $X$. The question thus is: how could we improve on that model to have bounded results with probabilities of default? A well known function which satisfies the inequality $0 \leq f(x) \leq 1$ (and is continuous and hence differentiable) is the so called "Sigmoid" or "Logistic" function which is defined as

$$\begin{equation}
S(x) = \frac{1}{1 + e^{-x}} = \frac{e^x}{1 + e^x}
\end{equation}$$

Above figure displays the function's shape. Its range is bounded by $[0, 1]$ and its notable "S" shape depends on the input parameter $x$. For logistic regression $x$ takes on the known functional form of a linear regression:

$$\begin{equation}
\Pr(Y=y|X) = \frac{e^{\beta_0 + \beta_1 X_1 + \ldots + \beta_p X_p}}{1 + e^{\beta_0 + \beta_1 X_1 + \ldots + \beta_p X_p}}
\end{equation}$$

This function is related to the linear model in that by rearranging the equation we arrive at

$$\begin{equation*}
\frac{p(X)}{1 - p(X)} = e^{\beta_0 + \beta_1 X_1 + \ldots + \beta_p X_p}
\end{equation*}$$

Notice that for ease of use we define $p(X)$ as equivalent of $\Pr(Y=y|X)$. Taking the natural logarithm of this expression yields

$$\begin{equation}
\underbrace{\log\left(\frac{p(X)}{1 - p(X)}\right)}_{\textit{Logit}} = \beta_0 + \beta_1 X_1 + \ldots + \beta_p X_p,
\end{equation}$$

a function where the output on the left-hand side, the so called **Logit** or **log-odds**, is linear in $X$.

## Estimating the Coefficients

To run a logistic regression we need to estimate the $\beta$ coefficients in above equation. In the linear regression setting we used a least squares approach. Here, however, we switch to a method called **maximum likelihood**. This approach is used in many areas of statistics/machine learning to fit models and as such often studied in graduate courses. Here we will restrict ourselves to a very superficial discussion of the intuition. For the interested reader, [Elkan (2014)](http://cseweb.ucsd.edu/~elkan/250B/logreg.pdf) provides an excellent, concise introduction, whereas Bishop (2006) has a more advanced, yet sound discussion of the topic.

Back to the intuition: maximum likelihood optimizes the Logit-equation to get estimates of $\beta_0, \beta_1$ "such that the predicted probability $\hat{p}(x_i)$ of default for each individual [$\ldots$] corresponds as closely as possible to the individual's observed default status. In other words, we try to find $\hat{\beta}_0$ and $\hat{\beta}_1$ such that plugging these estimates into the model for $p(X)$ [$\ldots$] yields a number close to one for all individuals who defaulted, and a number close to zero for all individuals who did not." (James et al. (2013, p. 133)).

The maximum likelihood function that is used to estimate the $\beta$ coefficients in the logistic regression model, is derived from the probability distribution of the dependent variable $y$. Since $y$ takes on one of two values (in our example $y \in \{0, 1\}$) and assuming the response values are independent of each other, the probability mass function of $Y$ follows a Bernoulli distribution, $Y \sim Bern(p)$, which is a special case of the Binomial distribution $Bin(n, p)$ with $n=1$. Its probability mass function is

\begin{equation*}
f(y;p) = p(Y=y) = p^y (1-p)^{1-y} =
\begin{cases}
p & \text{ if } y=1 \\
(1-p) & \text{ if } y=0
\end{cases}
\end{equation*}

The joint probability mass function of $Y$ is

\begin{equation*}
f(y | \beta) = \prod_{i=1}^N p(x_i)^{y_i} \, (1-p(x_i))^{1-y_i}
\end{equation*}

and describes the values of $y$ as a function of known, fixed values for $\beta$, where $\beta$ is related to $y$ through the logistic function. Since we don't know the coefficients $\beta$ but have measured outcomes for $y$ the likelihood function reverses above joint probability mass function such that it expresses the values of $\beta$ in terms of known, fixed values for $y$. This is the likelihood function ([Czepiel (2002)](https://czep.net/stat/mlelr.pdf)).

\begin{equation}
L(\beta | y) = \prod_{i=1}^N p(x_i)^{y_i} \, (1-p(x_i))^{1-y_i}
\end{equation}

The maximum likelihood estimates are now those values for $\beta$ which maximize the likelihood function $L(\beta | y)$. Typically, to find the maximum likelihood estimates we would differentiate the (log-) likelihood with respect to the coefficients, set the derivatives equal to zero, and solve. However, there is no closed-form solution for this and thus numerical methods (such as Newton-Raphson, Newton-conjugent gradient etc.) are required to derive a maxima ([Shalizi (2017)](http://www.stat.cmu.edu/~cshalizi/ADAfaEPoV/ADAfaEPoV.pdf)). Thankfully, statistical packages such as Python's `statsmodels` have these necessary tools integrated into the relevant functions such that we do not need to concern ourselves with the details.

## Logistic Regression in Python

### Package Selection

To show how logistic regression is run in Python we will again rely on functions from the `statsmodels` package. The Scikit-learn package `sklearn` contains similar functions. However, when it comes to calling results, the `statsmodels` functions for logistic regression follow those for linear regression which we got to know in the previous chapter. Therefore we will work with this package.

### Data Load

We will use the generic 'Default' data set from James et al. (2013), which we discussed in this chapter's introduction. It is taken from the book's corresponding `R` package and made available in this course's data folder on GitHub. We start our journey as usual with the initial load of the necessary packages and setting a few options.

```python
%matplotlib inline
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import statsmodels.api as sm
import statsmodels.formula.api as smf
plt.style.use('seaborn-whitegrid')
```

    C:\Anaconda\lib\site-packages\statsmodels\compat\pandas.py:56: FutureWarning: The pandas.core.datetools module is deprecated and will be removed in a future version. Please use the pandas.tseries module instead.
      from pandas.core import datetools

```python
# Default data set is not available online. Data was extracted from R package "ISLR"
df = pd.read_csv('Data/Default.csv', sep=',')

# Factorize 'No' and 'Yes' in columns 'default' and 'student'
df['defaultFac'] = df.default.factorize()[0]
df['studentFac'] = df.student.factorize()[0]
df.head(5)
```

      default
      student
      balance
      income
      defaultFac
      studentFac

      0
      No
      No
      729.526495
      44361.625074
      0
      0

      1
      No
      Yes
      817.180407
      12106.134700
      0
      1

      2
      No
      No
      1073.549164
      31767.138947
      0
      0

      3
      No
      No
      529.250605
      35704.493935
      0
      0

      4
      No
      No
      785.655883
      38463.495879
      0
      0

### Simple Logistic Regression

Similar to linear regression with `statsmodels` there exist different ways to compute a logistic regression. For reference, we will show three approaches here:

1. R-Style: The `statsmodels.formula.api` library has a `glm` function mimicking `R`'s way of running glm regressions
2. Classic GLM: Using the `GLM` function from the `statsmodels.api` library
3. Logit regression: `statsmodels.api`'s `Logit` function follows the approach we used for OLS case

You might ask what GLM/glm means. It stands for *Generlized Linear Models* and as such is a generalization of linear regression approaches (linear, logistic, Poisson regression). There are many good resources available on the web for the interested. A good textbook introducing GLM is Alan Agresti's *Foundations of Linear and Generalized Linear Models* (Agresti (2015)).

#### R-Style

* This method allows for a verbose definition of the regression.
* It follows the pattern `y ~ X1 + X2 + ... + Xn` - in words we regress $y$ on $X_1, X_2, \ldots, X_n$ where $X_i$ is a vector of a particular feature.
* By default a constant is added to the regression
* `family` is a necessary argument. Remember that above where we discussed the maximum likelihood approach of estimating the coefficients we mentioned that the binary response $y$ follows a Bernoulli distribution, which is a special case of a Binomial distribution. Thus we chose `family=sm.families.Binomial()`.
* The `.fit()` method runs the maximum likelihood estimation of the coefficients. Default method is 'Newton-Raphson' (method=`newton`) and this fits our needs. Nonetheless, others are available. Use the help function `sm.Logit.fit?` to see what options you have (it's the same for all three approaches).

More details can be found [here](http://www.statsmodels.org/dev/example_formulas.html)

```python
# 1. R-Style

# R-Style formula
formula = 'defaultFac ~ balance'

# Regress model
logReg = smf.glm(formula=formula, data=df, family=sm.families.Binomial()).fit()
print(logReg.summary())
```

                     Generalized Linear Model Regression Results
    ==============================================================================
    Dep. Variable:             defaultFac   No. Observations:                10000
    Model:                            GLM   Df Residuals:                     9998
    Model Family:                Binomial   Df Model:                            1
    Link Function:                  logit   Scale:                             1.0
    Method:                          IRLS   Log-Likelihood:                -798.23
    Date:                Mon, 19 Feb 2018   Deviance:                       1596.5
    Time:                        15:31:12   Pearson chi2:                 7.15e+03
    No. Iterations:                     9
    ==============================================================================
                     coef    std err          z      P>|z|      [0.025      0.975]
    ------------------------------------------------------------------------------
    Intercept    -10.6513      0.361    -29.491      0.000     -11.359      -9.943
    balance        0.0055      0.000     24.952      0.000       0.005       0.006
    ==============================================================================

#### Classic GLM

* `endog` = endogenous variable = response = $y$
* `exog` = exogenous variable(s) = features = $X$
* Notice that a constant needs to be added manually with `sm.add_constant(X)`

Function details are described [here](http://www.statsmodels.org/0.6.1/generated/statsmodels.genmod.generalized_linear_model.GLM.html).

```python
# 2. GLM

# Regress model
logReg = sm.GLM(endog=df.defaultFac, exog=sm.add_constant(df.balance),
                family=sm.families.Binomial()).fit()
print(logReg.summary())
```

                     Generalized Linear Model Regression Results
    ==============================================================================
    Dep. Variable:             defaultFac   No. Observations:                10000
    Model:                            GLM   Df Residuals:                     9998
    Model Family:                Binomial   Df Model:                            1
    Link Function:                  logit   Scale:                             1.0
    Method:                          IRLS   Log-Likelihood:                -798.23
    Date:                Mon, 19 Feb 2018   Deviance:                       1596.5
    Time:                        15:31:12   Pearson chi2:                 7.15e+03
    No. Iterations:                     9
    ==============================================================================
                     coef    std err          z      P>|z|      [0.025      0.975]
    ------------------------------------------------------------------------------
    const        -10.6513      0.361    -29.491      0.000     -11.359      -9.943
    balance        0.0055      0.000     24.952      0.000       0.005       0.006
    ==============================================================================

#### Logit Regression

* `endog` = endogenous variable = response = $y$
* `exog` = exogenous variable(s) = features = $X$
* Notice that a constant needs to be added manually with `sm.add_constant(X)`
* No need to specify `family` (Logit is per default binomial)

The full function details are to be found [here](http://www.statsmodels.org/dev/generated/statsmodels.discrete.discrete_model.Logit.html)

```python
# 3. Logit

# Regress model
logReg = sm.Logit(endog=df.defaultFac, exog=sm.add_constant(df.balance)).fit()
print(logReg.summary())
```

    Optimization terminated successfully.
             Current function value: 0.079823
             Iterations 10
                               Logit Regression Results
    ==============================================================================
    Dep. Variable:             defaultFac   No. Observations:                10000
    Model:                          Logit   Df Residuals:                     9998
    Method:                           MLE   Df Model:                            1
    Date:                Mon, 19 Feb 2018   Pseudo R-squ.:                  0.4534
    Time:                        15:31:13   Log-Likelihood:                -798.23
    converged:                       True   LL-Null:                       -1460.3
                                            LLR p-value:                6.233e-290
    ==============================================================================
                     coef    std err          z      P>|z|      [0.025      0.975]
    ------------------------------------------------------------------------------
    const        -10.6513      0.361    -29.491      0.000     -11.359      -9.943
    balance        0.0055      0.000     24.952      0.000       0.005       0.006
    ==============================================================================

    Possibly complete quasi-separation: A fraction 0.13 of observations can be
    perfectly predicted. This might indicate that there is complete
    quasi-separation. In this case some parameters will not be identified.

### Accessing Output

Though the scope of information in the summaries differ slightly between the different approaches the function yield the same results. As in the case of linear regression, the model's output can be accessed through its different methods and attributes. For example the models parameters and p-values are easily accessed as shown below.

```python
logReg.params
```

    const     -10.651331
    balance     0.005499
    dtype: float64

```python
logReg.pvalues
```

    const      3.723665e-191
    balance    2.010855e-137
    dtype: float64

### Hypothesis Testing

The $z$-statistic in above summary and the corresponding p-values play the same role as the $t$-statistic in the linear regression output. For example, the $z$-statistic associated with $\hat{\beta}_1$ is equal to $\hat{\beta}_1 / SE(\hat{\beta}_1)$. A large (absolute) $z$-value, and correspondingly a small p-value, provides evidence against the null hypothesis $H_0 : \beta_1 = 0$. Notice that this null hypothesis implies that $p(X) = \dfrac{e^{\beta_0}}{(1 + e^{\beta_0})}$.

The main purpose of the intercept is to adjust the average fitted probabilities to the proportion of ones in the data. Beyond that it is not of interest.

If we are interested in the coefficient's 99% CI (instead of 95% like in the summary) we can use the same code as in the linear regression case.

```python
logReg.conf_int(alpha=0.01)
```

      0
      1

      const
      -11.581640
      -9.721022

      balance
      0.004931
      0.006067

### Making Predictions

If we wish to make predictions of the default probability we can use the `.predict()` method. Given we leave the brackets empty, this method will calculate $p(X)$ on the basis of the previously used feature training sample. If we wish to get the probability of default for a balance of, let us say, USD 2'000 we can run the following line of code:

```python
# X must include 1 in first column for intercept
logReg.predict([1, 2000])
```

    array([ 0.58576937])

The vector $[1, 2000]$ in the above `.predict()` function corresponds to a row vector containing the $x$-values for which we wish to predict the probability of default. If we had $p$ features, this vector would obviously have lenght $p+1$ ($p$ features + 1 for intercept).

For future reference, below code generates a confusion matrix. We will discuss Confusion Matrices in more detail in the chapter on $k$-Nearest Neighbor. The `threshold` parameter defines above what probability an new observation is labeled as 1; default is 0.5.

```python
# Confusion Matrix for LogRegression
logReg.pred_table(threshold=0.5)
```

    array([[ 9625.,    42.],
           [  233.,   100.]])

### Plotting

Above we regressed default on balance. Thus this can still be displayed in a figure. In order to do so we need to sort the data first. Once this is done, plotting follows the usual routine.

```python
# Create sorted resutls-df
res = pd.DataFrame()
res['balance'] = df.balance
res['prob'] = logReg.predict()
res = res.sort_values('balance')

# Plot scatter & logReg
plt.figure(figsize=(12, 8))
plt.scatter(df.balance, df.defaultFac, marker='.')
plt.plot(res.balance, res.prob, c='k')
plt.axhline(y=0, color='gray', linestyle='dashed')
plt.axhline(y=1, color='gray', linestyle='dashed')
plt.ylabel('Probability of default', fontsize=12)
plt.xlabel('Balance', fontsize=12);
```

### Multiple Logistic Regression

Regressing 'default' on multiple features follows the same procedure as for simple logistic regression. As a case of use we regress 'default' on 'balance', a credit card holder's 'income' and the dummy variable 'studentFac' (1 if yes, 0 otherwise).

```python
# Assign features to matrix X and response to y
X = sm.add_constant(df[['balance', 'income', 'studentFac']])
X.income = X.income / 1000  # Income in 1'000 dollars
y = df.defaultFac

# Run Logistic Regression
logReg = sm.Logit(endog=y, exog=X).fit()
print(logReg.summary().tables[1])
```

    Optimization terminated successfully.
             Current function value: 0.078577
             Iterations 10
    ==============================================================================
                     coef    std err          z      P>|z|      [0.025      0.975]
    ------------------------------------------------------------------------------
    const        -10.8690      0.492    -22.079      0.000     -11.834      -9.904
    balance        0.0057      0.000     24.737      0.000       0.005       0.006
    income         0.0030      0.008      0.370      0.712      -0.013       0.019
    studentFac    -0.6468      0.236     -2.738      0.006      -1.110      -0.184
    ==============================================================================

The p-value for `income` is surprisingly large, indicatign that there is no clear evidence of a real association between the probability of default and income. In a case study this certainly would need to be further examined.

### Multiple Logistic Regression with Scikit-learn

For reference we show below how logistic regression is run with Scikit-learn. Unfortunately, there is no summary output available in Scikit-learn like there is within the Statsmodels package. The main reason is that sklearn is not used for statistical inference but for predictive modelling/ML and the evaluation criteria are based on performance on previously unseen data.

Notice that in order to have similar coefficients/results, the hyperparameter `C` had to be set to `C=1e9`. This parameter is a regularization parameter that refers to the L1/L2 penalty scheme implemented. For details refer to Scikit-learn's [user guide](http://scikit-learn.org/stable/modules/linear_model.html#logistic-regression), [this discussion on Stackoverflow](https://stackoverflow.com/questions/43431236/how-to-perform-an-unregularized-logistic-regression-using-scikit-learn) or chapter 6.2 in James et al. (2013).

```python
from sklearn.linear_model import LogisticRegression
logReg = LogisticRegression(C=1e9, fit_intercept=False).fit(X, y)
```

```python
logReg.coef_
```

    array([[ -1.08564927e+01,   5.73008186e-03,   2.97669622e-03,
             -6.46873934e-01]])

### Decision Boundary

We've seen above how one can plot a logistic regression in the simple case with just one feature. If you regress a response on two features, it is still possible to create very meaningful plots. A good example is [this answer on stackoverflow](https://stackoverflow.com/questions/28256058/plotting-decision-boundary-of-logistic-regression) where the decision boundary is drawn and the color scheme follows the probability of the response.

## Closing Comments
In closing this chapter it should be mentioned that the preceding code sections are by no means a thorough analysis but a superficial digging into the topics. Yet the tools and measures necessary to understand the output are yet to be introduced. For this we refer to the chapter on linear discriminant analysis (LDA) and quadratic discriminant analysis (QDA) (and subsequent chapters) where many tools are introduced that are of significant help in analyzing an algorithm's output.

# Further Ressources

In writing this notebook, many ressources were consulted. For internet ressources links are provided within the textflow above. Beyond these links, the following ressources were consulted and are recommended as further reading on the discussed topics:

* Agresti, Alan, 2015, *Foundations of Linear and Generalized Linear Models* (John Wiley & Sons, Hoboken, NJ).
* Bishop, Christopher M., 2006, *Pattern Recognition and Machine Learning* (Springer, New York, NY).
* Czepiel, Scott A., 2002, Maximum Likelihood Estimation of Logistic Regression Models: Theory and Implementation from website, https://czep.net/stat/mlelr.pdf, 08/24/17.
* Elkan, Charles, 2014, Maximum Likelihood, Logistic Regression, and Stochastic Gradient Training from website, http://cseweb.ucsd.edu/~elkan/250B/logreg.pdf, 08/24/17.
* James, Gareth, Daniela Witten, Trevor Hastie, and Robert Tibshirani, 2013, *An Introduction to Statistical Learning: With Applications in R* (Springer Science & Business Media, New York, NY).
* Shalizi, Cosma Rohilla, 2017, Advanced Data Analysis from an Elementary Point of View from website, http://www.stat.cmu.edu/~cshalizi/ADAfaEPoV/ADAfaEPoV.pdf, 08/24/17.


```python
# HIDDEN
from datascience import *
from prob140 import *
import numpy as np
import matplotlib.pyplot as plt
plt.style.use('fivethirtyeight')
%matplotlib inline
from scipy import stats
```

### Calculus in SymPy ###
Working with densities involves calculus which can sometimes be time-consuming. In this course we will give you two ways of reducing the amount of calculus involved.

- Probabilistic methods can help reduce algebra and calculus. You've seen this with algebra in the discrete case. You'll see it with calculus as we learn more about densities.
- Python has a symbolic math module called `SymPy` that does algebra, calculus, and much other symbolic math. In this section we will show you how to do calculus using `SymPy`.

We will demonstrate the methods in the context of an example. Suppose $X$ has density given by

$$
f(x) =
\begin{cases}
105x^2(1-x)^4 ~~~ \text{if } 0 \le x \le 1 \\
0 ~~~~~~~~~ \text{otherwise}
\end{cases}
$$

As you can see from its graph below, $f$ could be used to model the distribution of a random proportion that you think is likely to be somewhere between 0.2 and 0.4.

```python
# HIDDEN
x_vals = np.arange(0, 1.01, 0.01)
f = 105 * x_vals**2 * (1 - x_vals)**4
plt.plot(x_vals, f, color='darkblue', lw=2)
plt.xlabel('$x$')
plt.ylabel('$f(x)$', rotation=0);
```

The density $f$ is a polynomial on the unit interval, and in principle the algebra and calculus involved in integrating it are straightforward. But they are tedious. So let's get `SymPy` to do the work.

First, we will import all the functions in `SymPy` and set up some printing methods that make the output look nicer than the retro typewritten pgf output you saw in a previous section. In future sections of this text, you can assume that this importing and initialization will have been done at the start.

```python
from sympy import *
init_printing()
```

Next, we have to tell Python which variables are symbolic and what their possible values are. The function `declare` lets us do this. It takes as its arguments the string representing the variable, and an option `interval` specifying the interval of possible values of the variable. In our example, the variable `x` takes values in the unit interval. In later examples we will show you how to declare infinite intervals of possible values.

```python
declare('x', interval=(0, 1))
```

Now we will assign the name `density` to the expression that defines $f$. The expression looks just like a numerical calculation, but the output is algebraic!

```python
density = 105 * x**2 * (1-x)**4
density
```

That's the density $f$ defined by the equation at the start of the section. Notice that what we naturally think of as $1 - x$ is expressed as $-x + 1$. That's because `SymPy` is writing the polynomial leading with the term of highest degree.

Let's not simply accept that this function is a density. Let's check that it is a density by integrating it from 0 to 1. To do this, we use the method `Integral` that takes the name of a function and a *tuple* (a sequence in parentheses) consisting of the variable of integration and the lower and upper limits of integration. We have assigned this integral to the name `total_area`

```python
total_area = Integral(density, (x, 0, 1))
total_area
```

The output of displays the integral, which is nice, but what we really want is its numerical value. In `SymPy`, this is achieved by rather rudely instructing the method to `doit()`.

```python
total_area.doit()
```

This confirms that the function $f$ is a density.

We can use `Integral` again to find the chance of any interval. Here is $P(0.2 < X < 0.4)$.

```python
p_02_04 = Integral(density, (x, 0.2, 0.4)).doit()
p_02_04
```

For $x$ in the unit interval, the cdf of $X$ is
$$
F(x) ~ = ~ P(X \le x) ~ = ~ \int_0^x f(s)ds ~ = ~ I(s)~ \Big{\rvert}_0^x ~ = ~ I(x) - I(0)
$$

where $I$ is the indefinite integral of $f$.

To get the indefinite integral, simply ask `SymPy` to integrate the density; there are no limits of integration.

```python
indefinite = Integral(density).doit()
indefinite
```

Now $F(x) = I(x) - I(0)$. You can see at a glance that $I(0) = 0$ but here is how `SymPy` would figure that out.

To evaluate $I(0)$, `SymPy` must substitute $x$ with 0 in the expression for $I$. This is achieved by the method `subs` that takes the variable as its first argument and the specified value as the second.

```python
I_0 = indefinite.subs(x, 0)
I_0
```

```python
cdf = indefinite - I_0
cdf
```

To find the value of the cdf at a specified point, say 0.4, we have to substitute $x$ with 0.4 in the formula for the cdf.

```python
cdf_04 = cdf.subs(x, 0.4)
cdf_04
```

Thus $P(X \le 0.4)$ is roughly 58%. Earlier we calulated $P(0.2 < X < 0.4) = 43.2\%$, which we can confirm by using the cdf:

```python
cdf_02 = cdf.subs(x, 0.2)
cdf_04 - cdf_02
```

The expectation $E(X)$ is a definite integral from 0 to 1:

```python
expectation = Integral(x*density, (x, 0, 1)).doit()
expectation
```

Notice how simple the answer is. Later in the course, you will see why.

Here is $E(X^2)$, which turns out to be another simple fraction. Clearly, the density $f$ has interesting properties. We will study them later. For now, let's just get the numerical answers.

```python
expected_square = Integral((x**2)*density, (x, 0, 1)).doit()
expected_square
```

Now you can find $SD(X)$.

```python
sd = (expected_square - expectation**2)**0.5
sd
```

### SymPy and the Exponential Density ###
One of the primary distributions in probability theory, the exponential distribution has a positive parameter $\lambda$ and density given by

$$
f(t) ~ = \lambda e^{-\lambda t}, ~~~ t \ge 0
$$

The density is 0 on the negative numbers. Here is its graph when $\lambda = 3$.

```python
# HIDDEN

lam = 3
tt = np.arange(0, 2.01, 0.01)
f = lam * np.exp(-1*lam*tt)
plt.plot(tt, f, color='darkblue', lw=2)
plt.xlabel('$t$')
plt.ylabel('$f(t)$', rotation=0)
plt.title('Exponential Density, $\lambda = 3$');
```

To check that $f$ is a density, we have to confirm that its integral is 1. So we will declare two positive symbolic variables `t` and `lamda`. Notice the incorrectly spelled `lamda`. That is because `lambda` has another meaning in Python, as some of you might know.

In fact `lamda` is a constant, not a variable. But `SymPy` needs to know that it's an algebraic object, so we have to declare it as such.

Note the use of `positive=True` to declare positive variables.

```python
declare('lamda', positive=True)
declare('t', positive=True)
```

Now we will define the density function. Notice the use of `exp` for the exponential function. Notice also that the form of the answer looks different from the way we have written it above, though it's algebraically the same.

```python
expon_density = lamda * exp(-lamda * t)
expon_density
```

This is an unavoidable aspect of computer aided algebra, and it is the reason we will use `SymPy` purely for computation, not for display.

To see that the function is a density, we can check that its integral from 0 to $\infty$ is 1. The symbol that `SymPy` uses for $\infty$ is a double lower case o: `oo`. That looks very much like $\infty$.

```python
Integral(expon_density, (t, 0, oo)).doit()
```

Suppose $T$ has the exponential $(\lambda)$ density. Then for $t \ge 0$ the cdf of $T$ is

$$
F_T(t) ~ = ~ P(T \le t) ~ = ~ \int_0^t \lambda e^{-\lambda s}ds
$$

This is a straightforward integral that you can probably do in your head. However, let's get some more practice using `SymPy` to find cdf's.
We will use the same method that we used to find the cdf in the previous example.

$$
\int_0^t \lambda e^{-\lambda s}ds ~ = ~ I(t) - I(0)
$$

where $I$ is the indefinite integral of the density. To get this indefinite integral we will use `Integral` as before, except that this time we must specify `t` as the variable of integration. That is because `SymPy` sees two algebraic quantities `t` and `lamda` in the density, and doesn't know which one is the variable unless we tell it.

```python
indefinite = Integral(expon_density, t).doit()
indefinite
```

Now use $F_T(t) = I(t) - I(0)$:

```python
I_0 = indefinite.subs(t, 0)
I_0
```

```python
cdf = indefinite - I_0
cdf
```

Thus the cdf of the exponential $(\lambda)$ density is
$$
F_T(t) ~ = ~ 1 - e^{-\lambda t}
$$
Also,
$$
E(T) ~ = ~ \int_0^\infty t \lambda e^{-\lambda t} dt ~ = ~ \frac{1}{\lambda}
$$

which you can check by integration by parts. But `SymPy` is faster:

```python
expectation = Integral(t*expon_density, (t, 0, oo)).doit()
expectation
```

And
$$
E(T^2) = \frac{2}{\lambda^2}
$$

```python
expected_square = Integral(t**2 * expon_density, (t, 0, oo)).doit()
expected_square
```

So
$$
Var(T) ~ = ~ \frac{2}{\lambda^2} - \frac{1}{\lambda^2} ~ = ~ \frac{1}{\lambda^2}
$$
and hence
$$
SD(T) ~ = ~ \frac{1}{\lambda}
$$

The purpose of this section has been to give you a workout in `SymPy`. We will take a closer look at the exponential distribution in the next section.

```python

```


# **Optimización - Actividad 3**

*   Estudiante: Alejandro Jesús Manotas Marmolejo
*   Código: 200108289

# **Pregunta 1.**

Considere $N$ funciones convexas $f_i(x):\Re\Rightarrow \Re$, para $1 \le i \le N$, demuestre que su suma
\begin{eqnarray}
\sum_{i=1}^{N} f_i(x)\,,
\end{eqnarray}
es convexa.

**Respuesta.**

\begin{equation}
\begin{split}
& x_1, x_2 \in \Re\\
& z = \alpha x_1 + (1-\alpha)x_2\\
& f(z) = g(\alpha x_1 + (1-\alpha)x_2)\\
& g(x) = \sum_{i=1}^{N} f_i(x)\\
& = \sum_{i=1}^{N} f_i(z)\\
& = \sum_{i=1}^{N} f(\alpha x_1 + (1-\alpha)x_2)\\
& = \sum_{i=1}^{N} \alpha f_i(x_1) + (1-\alpha)f_i(x_2)\\
& = \sum_{i=1}^{N} \alpha f_i(x_1) + \sum_{i=1}^{N} (1-\alpha)f_i(x_2)\\
& = \alpha \sum_{i=1}^{N} f_i(x_1) + (1-\alpha)\sum_{i=1}^{N} f_i(x_2)\\
& = \alpha g(x_1) + (1-\alpha) g(x_2)
\end{split}
\end{equation}

---

# **Pregunta 2**

Demuestre que $g(x) = a\cdot f(x) +b$ es convexa, en donde $a,b\ge 0$ y $f(x):\Re\Rightarrow \Re$ es una función convexa.

**Respuesta.**

\begin{equation}
\begin{split}
& x_1, x_2 \in \Re\\
& z = \alpha x_1 + (1-\alpha)x_2\\
& f(z) = g(\alpha x_1 + (1-\alpha)x_2)\\
& g(x) = a \cdot f(x) + b\\
& = a \cdot f(z) + b \\
& = a \cdot (f(\alpha x_1 + (1-\alpha)x_2)) + b\\
& = a \cdot (\alpha f(x_1) + (1-\alpha) f(x_2)) + b\\
& = a\alpha f(x_1) + (1-\alpha)af(x_2) + b \\
& = a\alpha f(x_1) + (1-\alpha)af(x_2) + b + \alpha b - \alpha b\\
& = a\alpha f(x_1) + \alpha b + (1-\alpha)af(x_2) + b - \alpha b\\
& = \alpha (a f(x_1) + b) + (1-\alpha)af(x_2) + b (1 - \alpha)\\
& = \alpha (a f(x_1) + b) + (1-\alpha)af(x_2) + b (1 - \alpha)\\
& = \alpha [a f(x_1) + b] + (1-\alpha) [af(x_2) + b]\\
& = \alpha g(x_1) + (1-\alpha) g(x_2)
\end{split}
\end{equation}

---

# **Pregunta 3**

Demuestre que, si $f(x):\Re \Rightarrow \Re$ es convexa, entonces $g(x) = f(a\cdot x +b)$ es convexa también, en donde $a,b \ge 0$.

**Respuesta.**

\begin{equation}
\begin{split}
& x_1, x_2 \in \Re \\
& z = \alpha x_1 + (1-\alpha) x_2\\
& f(z) = g(\alpha x_1 + (1-\alpha)x_2)\\
& g(x) = f(a \cdot x + b)\\
& = f(a \cdot (\alpha x_1 + (1-\alpha)x_2) + b)\\
& = f(a \alpha x_1 +  (1-\alpha)ax_2 + b)\\
& = f(a \alpha x_1 +  (1-\alpha)ax_2 + b + \alpha b - \alpha b)\\
& = f(a \alpha x_1 + \alpha b +  (1-\alpha)ax_2 + b - \alpha b)\\
& = f(\alpha [a x_1 + b] +  (1-\alpha)ax_2 + b [1- \alpha])\\
& = f(\alpha [a x_1 + b] +  (1-\alpha) [ax_2 + b])\\
& = \alpha f([a x_1 + b]) +  (1-\alpha) f(ax_2 + b)\\
& = \alpha g(x_1) +  (1-\alpha) g(x_2)\\
\end{split}
\end{equation}

---

**Recuerde:** En este curso no se tolerará el plagio. Sin excepción, en caso de presentarse esta situación, a los estudiantes involucrados se les iniciará proceso de investigación, y se actuará en conformidad con el Reglamento de Estudiantes de la Universidad del Norte. El plagio incluye: usar contenidos sin la debida referencia, de manera literal o con mínimos cambios que no alteren el espíritu del texto/código; adquirir con o sin intención, trabajos de terceros y presentarlos parcial o totalmente como propios; presentar trabajos en grupo donde alguno de los integrantes no trabajó o donde no se hubo trabajo en equipo demostrable; entre otras situaciones definidas en el manual de fraude académico de la Universidad del Norte:

(https://guayacan.uninorte.edu.co/normatividad_interna/upload/File/Guia_Prevencion_Fraude%20estudiantes(5).pdf )

**Isaías 40:31.** pero los que esperan a Jehová tendrán nuevas fuerzas; levantarán alas como las águilas; correrán, y no se cansarán; caminarán, y no se fatigarán.


#  Exercise 11

##  Introduction to Programming and Numerical Analysis

####
 Sebastian Honoré

## Plan for today

1. Welcome
2. Linear algebra in python
3. Solving equations analytically
4. Problemset 6

**Remember peer feedback deadline is Sunday 24th by 23.59**

# 2.1 Linear algebra in python

We can use `SciPy's` module `linalg` to perform a range of operations within the realm of linear algebra:

- Basic stuff: determinant, invert, norm
- Matrix decompositions (LU, Cholesky etc.)
- Solve a system of equations
- Find eigenvalues

# 2.2 Solve a system of equations
Lets take the module for a spin. We want to solve:

$$Ax=b$$

```python
import numpy as np
from scipy import linalg
np.random.seed(2021)
A = np.random.uniform(size=(5,5))
b = np.random.uniform(size=5)
print(f'Matrix A: \n{A} \n\n Matrix b:\n{b}')
```

    Matrix A:
    [[0.60597828 0.73336936 0.13894716 0.31267308 0.99724328]
     [0.12816238 0.17899311 0.75292543 0.66216051 0.78431013]
     [0.0968944  0.05857129 0.96239599 0.61655744 0.08662996]
     [0.56127236 0.61652471 0.96384302 0.57430429 0.37116085]
     [0.45214524 0.20185025 0.56930512 0.19509597 0.58370402]]

     Matrix b:
    [0.47631347 0.5178144  0.82309863 0.73222503 0.06905627]

# 2.3 Solve a system of equations

There are several ways we can solve this system of equations:

```python
# Using a regular solver ()
x = linalg.solve(A,b)
x
```

    array([ 3.17983125, -1.88260751, -2.05057824,  4.3774516 , -1.15692713])

```python
# Using LU decomposition
#-> decompose A into upper and lower triangular matrix and solve through substitution
LU, piv = linalg.lu_factor(A)
x = linalg.lu_solve((LU,piv),b)
x
```

    array([ 3.17983125, -1.88260751, -2.05057824,  4.3774516 , -1.15692713])

# 2.4 When to use solve and lu_factorization?

Combined linalg.lu_factor and linalg.lu_solve is equivalent to linalg.solve. But the algorithmic complexity is different!

- linalg.solve is $O(n^3)$, but linalg.lu_solve is $O(n^2)$
- So when only the b matrix changes and the A matrix is constant it is more efficient to use linalg.lu_solve.
- If the matrix is high dimensional linalg.lu_solve is fastest.

# 2.4 Why care?

Being able to solve a system of linear equations probably won't earn much street credit from your friends. However, it may come in handy. Remember the good ol' OLS equation? In case you forgot:

$$\hat{\beta} = (X'X)^{-1}X'y$$

In the 2020 exam you had to code the estimator from scratch and solve it using the functionalities of linalg. How?

```python
# Solving OLS
def beta_hat(x1,x2,y,con=True):
    """
    Estimates the vector of coefficients for two independent variables

    Args:
    x1 (array): Array of independent variable values
    x2 (array): Array of independent variable values
    y (array): Array of dependent variable values
    con (bool): Include array of intercepts. Defaults to True.

    Returns:
    beta_hat (float): vector of estimated coefficients
    """

    if con == True:
        con = np.ones((x1.shape)) #if con=True create a N x 1 vector, for the intercepts to be estimated
        X = np.stack((con,x1,x2),axis=1) #Stack vectors into the matrix X, as shown above
    else:
        X = np.stack((x1,x2),axis=1) #Stack vectors into the matrix X, as shown above

    beta_hat = linalg.inv(X.T@X)@X.T@y #The matrix of coeffients is computed given the definition
    return beta_hat
```

# 3.1 Solving equations analytically
Formulas and models are part of the life as an economist. `Sympy` enables us to translate these formulas into python code that we can work with - super cool! Lets see it in action:

Consider a utility function from a standard OLG model. Economic agents lives two periods (young/old) and obtains utility from consumption in both periods:

$$U_{i,t} = u_i(c_{i,1}) + \frac{1}{1+\rho}u_i(C_{i,2})$$

# 3.2 Implementation in Sympy
For the sake of simplicity let's assume log-utility:

```python
import sympy as sm
c1,c2 = sm.symbols("C_i1"), sm.symbols("C_i2")
rho = sm.symbols("rho")

#utility
uc1 = sm.ln(c1)
uc2 = sm.ln(c2)
U = uc1+1/(1+rho)*uc2
U
```

$\displaystyle \log{\left(C_{i1} \right)} + \frac{\log{\left(C_{i2} \right)}}{\rho + 1}$

# 3.3 Sympy functionalities
We can do a lot of stuff from here. Say we need the derivative of $U$ with respect to $C_{i,2}$

```python
sm.diff(U,c2)
```

$\displaystyle \frac{1}{C_{i2} \left(\rho + 1\right)}$

**Differentation has never been easier**

# 3.4 Turn your formulas into functions
A cool feature of Sympy is that you can turn your formulas into python functions. This will likely be useful in your model project:

- Solve model analytically with Sympy
- Convert solution into python function e.g. the law-of-motion in the OLG model
- Find steady state using an optimizer

How is it done?

# 3.5 Functions in Sympy

Use the `Lambdify` method which takes a function and an iterable as argument. In our case the function is utility and the iterable is consumption:

```python
util = sm.lambdify((c1,c2,rho),U)

print(f'Utility for C1=10, C2=5 and rho=0.1: {util(10,5,0.1):.2f}')
```

    Utility for C1=10, C2=5 and rho=0.1: 3.77

# Problemset 6

Your turn to work with `linalg` and `SymPy`.


 $$\left\{ \begin{array}{lcc}
            \dot{x}_{1}=-x_{1}^3 + u\\
             \\ \dot{x}_{2}=x_{1}
             \end{array}
\right.$$

```python
import sympy as sym
```

```python
#Con esto las salidas van a ser en LaTeX
sym.init_printing(use_latex=True)
```

```python
x_1, x_2 ,theta = sym.symbols('x_1 x_2 theta')
```

```python
X = sym.Matrix([x_1, x_2])
X
```

```python
f_1 = -x_1**3 - x_2
```

```python
f_2 = x_1
```

```python
F = sym.Matrix([f_1,f_2])
F
```

```python
# puntos de equilibrio del sistema
pes = sym.solve([f_1,f_2])
pes
```

```python
A = F.jacobian(X)
A
```

```python
A_1 = A.subs({x_1:0,x_2:0})
A_1
```

```python
A_1.eigenvals()
```

```python

```


パラメータの増減に際しいちいちLossの再計算をするのはめんどう
`autograd`パッケージは、自動微分の機能を提供することでこれを低減できる
他の多くのパッケージでは自動微分にはコンパイルが必要だがmxnetではpytorchのように、宣言的に、通常のコード内に記述可能

```python
import mxnet as mx
import mxnet.ndarray  as nd
import mxnet.autograd as ag # 0.10.0 （一応最新版）でもmxnet直下へはまだ取り込まれていない模様
```

# Attaching gradients

$$
f = 2x^2
$$
のxについての微分について考える

```python
x = mx.nd.array([[1,2],[3,4]])
```

一度fのxについての傾きを計算すると、どこかにそれを保持しておく必要がある
mxnetでは`attach_grad()`メソッドで起動できる

```python
x.attach_grad()
```

この上でfを定義するとMXNetは計算グラフを自動生成する…あたかもレコーディングマシンを起動し、変数が生成された正しい経路をキャプチャするかのように
計算グラフの生成には少なくない計算資源が必要となるため、明示的に宣言しないとMXNetはグラフを生成しない
`with autograd.record()`によりレコーディングを開始する

```python
with ag.record():
    y = x * 2
    z = y * x
```

`z.backward()`で逆伝播の計算を行う
zが複数のエントリーを持つとき、`z.backward()`は`mx.nd.sum(z).backward()` に相当する

```python
z.backward()
```

$$
\begin{align}
y = 2x\\
z = xy
\end{align}
$$

なので

$$
z = 2x^2
$$

$$
\begin{align}
\frac{dy}{dx} = 2 \\
\frac{dz}{dx} = 4x
\end{align}
$$

```python
print(x.grad)

```

    [[  4.   8.]
     [ 12.  16.]]
    <NDArray 2x2 @cpu(0)>

# Head gradients and the chain rule
yがxの関数であるとき、xに対するyの導関数のみに関心がある。
一方でzがxの関数であるときは、xに対するzの傾きのみに関心がある。
連鎖則を用いると（TutorialだとTrain ruleになってる…雑）

$$
\frac{dz}{dx} = \frac{dz}{dy} \frac{dy}{dx}
$$

yはzの一部と言え、`x.grad`にdz/dxを格納したいとき、先頭の傾きとしてdz/dyを`backward()`の入力として渡すことができる。デフォルトの引数は`nd.ones_like(y)`となる

```python
with ag.record():
    y = x * 2
    z = y * x

head_gradient = nd.array([[10,1.],[.1,.01]])
z.backward(head_gradient)
print(x.grad)
```

    [[ 40.           8.        ]
     [  1.20000005   0.16      ]]
    <NDArray 2x2 @cpu(0)>

> わかりづらい原因はこれまでの説明文: xでの傾き(4,8,12,16)について、重み付けの行列(10,1,0.1,0.01)を渡せるよ、ということでいいのだと思う（多分） 何も指定しない場合は1が初期値になるという話

基礎がわかったところで、Pythonicな制御フローでの計算例

```python
a = nd.random_normal(shape=3) # 1行3列、正規分布で初期化された配列
a.attach_grad()

with ag.record():
    b = a * 2
    while (nd.norm(b) < 1000).asscalar():  # 行列のL2ノルムが1000以下
        b = b * 2

    if (mx.nd.sum(b) > 0).asscalar():
        c = b
    else:
        c = 100 * b
```

```python
head_gradient = nd.array([0.01,1.0,.1])
c.backward(head_gradient)
```

```python
print(a.grad)  # チュートリアルだとx.gradを出力してる…
```

    [   5.11999989  512.           51.20000076]
    <NDArray 3 @cpu(0)>

```python
print(a)
print(b)
print(c)
```

    [ 2.21220636  1.16307867  0.7740038 ]
    <NDArray 3 @cpu(0)>

    [ 1132.6496582    595.49627686   396.28994751]
    <NDArray 3 @cpu(0)>

    [ 1132.6496582    595.49627686   396.28994751]
    <NDArray 3 @cpu(0)>

```python
nd.norm(b)

```

    [ 1339.61071777]
    <NDArray 1 @cpu(0)>

c = 2^T a なのでだいたいこんな感じになる:512や1024などなど（Pythonic...なのか？）

notebooks/01_crashcourse/Automatic differentiation with autograd.ipynb


# Maximizing Concentration of an Intermediate in a Batch Reactor

A desired product $B$ is produced as intermediate in a series reaction

\begin{align}
A \overset{k_A}{\longrightarrow} B \overset{k_B}{\longrightarrow} C
\end{align}

where $A$ is a raw material and $C$ is a undesired b
y-product. The reaction operates at temperature where the rate constants are $k_A = 0.5\ \mbox{min}^{-1}$ and $k_A = 0.1\ \mbox{min}^{-1}$. The raw material is available as a solution with concenration $C_{A,f} = 2.0\ \mbox{moles/liter}$.

A 100 liter tank is available which can be operated as a batch reactor. At what point should the operation be terminated in order to maximize the concentration of $B$?

## Mathematical Model

A material balance for an isothermal stirred batch reactor with a volume $V = 40$ liters and an initial concentration $C_{A,f}$ is given by

\begin{align}
V\frac{dC_A}{dt} & = - V k_A C_A \\
V\frac{dC_B}{dt} & = V k_A C_A - V k_B C_B
\end{align}

Eliminating the common factor $V$

\begin{align}
\frac{dC_A}{dt} & = - k_A C_A \\
\frac{dC_B}{dt} & = k_A C_A - k_B C_B
\end{align}

With an initial concentration $C_{A,f}$. A numerical solution to these equations is shown in the following cell.

```python
%matplotlib inline
import matplotlib.pyplot as plt
import numpy as np
from scipy.integrate import odeint

V = 40     # liters
kA = 0.5   # 1/min
kB = 0.1   # l/min
CAf = 2.0  # moles/liter

def batch(X, t):
    CA, CB = X
    dCA_dt = -kA*CA
    dCB_dt = kA*CA - kB*CB
    return [dCA_dt, dCB_dt]

t = np.linspace(0,30,200)
soln = odeint(batch, [CAf,0], t)
plt.plot(t, soln)
plt.xlabel('time / minutes')
plt.ylabel('concentration / moles per liter')
plt.title('Batch Reactor')
plt.legend(['$C_A$','$C_B$'])
```

## Optimization with `scipy.minimize_scalar`

To find the maximum value, we first write a function to compute $C_B$ for any value of time $t$.

```python
def CB(tf):
    soln = odeint(batch, [CAf, 0], [0, tf])
    return soln[-1][1]
```

We gain use `minimize_scalar` to find the value of $t$ that minimizes the negative value of $C_B(t)$.|

```python
from scipy.optimize import minimize_scalar
minimize_scalar(lambda t: -CB(t), bracket=[0,50])
```

         fun: -1.3374806339222158
        nfev: 20
         nit: 19
     success: True
           x: 4.0235949243406663

```python
tmax = minimize_scalar(lambda t: -CB(t), bracket=[0,50]).x

print('Concentration c_B has maximum', CB(tmax), 'moles/liter at time', tmax, 'minutes.')
```

    Concentration c_B has maximum 1.33748063392 moles/liter at time 4.02359492434 minutes.

## Solution Using Pyomo

The variable to be found is the time $t_f$ corresponding to the maximum concentration of $B$. For this purpose we introduce a scaled time

$$\tau = \frac{t}{t_f}$$

so that $\tau=1$ as the desired solution. The problem then reads

\begin{align}
\max_{t_f} C_B(\tau=1)
\end{align}

subject to

\begin{align}
\frac{dC_A}{d\tau} & = - t_f k_A C_A \\
\frac{dC_B}{d\tau} & = t_f(k_A C_A - k_B C_B)
\end{align}

The solution to this problem is implemented as a solution to the following Pyomo model.

```python
from pyomo.environ import *
from pyomo.dae import *

V   = 40    # liters
kA  = 0.5   # 1/min
kB  = 0.1   # l/min
cAf = 2.0   # moles/liter

m = ConcreteModel()

m.tau = ContinuousSet(bounds=(0, 1))

m.tf = Var(domain=NonNegativeReals)
m.cA = Var(m.tau, domain=NonNegativeReals)
m.cB = Var(m.tau, domain=NonNegativeReals)

m.dcA = DerivativeVar(m.cA)
m.dcB = DerivativeVar(m.cB)

m.odeA = Constraint(m.tau,
    rule=lambda m, tau: m.dcA[tau] == m.tf*(-kA*m.cA[tau]) if tau > 0 else Constraint.Skip)
m.odeB = Constraint(m.tau,
    rule=lambda m, tau: m.dcB[tau] == m.tf*(kA*m.cA[tau] - kB*m.cB[tau]) if tau > 0 else Constraint.Skip)

m.ic = ConstraintList()
m.ic.add(m.cA[0]  == cAf)
m.ic.add(m.cB[0]  == 0)

m.obj = Objective(expr=m.cB[1], sense=maximize)

TransformationFactory('dae.collocation').apply_to(m)
SolverFactory('ipopt').solve(m)
print('Concentration c_B has maximum', m.cB[1](), 'moles/liter at time', m.tf(), 'minutes.')
```

    Concentration c_B has maximum 1.337480581022107 moles/liter at time 4.023594178375687 minutes.

```python

```