```python
import numpy as np
import pandas as pd
pd.options.mode.chained_assignment = None
from matplotlib import pyplot as plt
%matplotlib inline
import random

from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error
from sklearn.preprocessing import StandardScaler, PolynomialFeatures
from sklearn.linear_model import LinearRegression, LogisticRegression
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor
from scipy.stats import norm, sem
from scipy.interpolate import UnivariateSpline
from sklearn.calibration import calibration_curve
import matplotlib.pyplot as plt
import seaborn as sns
from scipy.stats.stats import pearsonr
from numpy.random import default_rng

import seaborn as sns
from pycebox.ice import ice, ice_plot
from matplotlib.cm import seismic

from sympy import *
```

```python
x1,x2,x3,y = [],[],[],[]
N = 1000

for i in range(0,N):
    X1 = np.random.uniform(-1, 1, size = 1)
    X2 = np.random.uniform(-1, 1, size = 1)
    X3 = np.random.uniform(-1, 1, size = 1)
    Y = 0.2*X1 - 5*X2 + 10*X3

    x1.append(X1)
    x2.append(X2)
    x3.append(X3)
    y.append(Y)
```

```python
data = np.concatenate([x1,x2,x3,y], axis = 1)
df = pd.DataFrame(data, columns=['X1','X2','X3','Y'])
```

```python
df
```

```python
X = df[['X1','X2','X3']]
X1, X2, X3 = df.X1, df.X2, df.X3
y = df.Y

std = {}
```

```python
minus_X, plus_X, minus_y, plus_y = train_test_split(X, y, test_size=.5, random_state=42)
minus_df = minus_X.join(minus_y)
plus_df = plus_X.join(plus_y)
```

```python
def change_features(X_og,X,i,alpha,group):
    X_list = []
    y_list = []

    for j in range(len(X)):
        new_values = {}
        std[i] = np.std(X[i])
        og_value = X[i].iloc[j]
        if group == 'plus':
            new_value = og_value + alpha*std[i]
        if group == 'minus':
            new_value = og_value - alpha*std[i]
        new_values[i] = new_value
        X_list.append(new_values)
    X_new = pd.DataFrame(X_list)
    X_new = X_new.join(X.reset_index().drop(columns=[i,'index']))
    for x in range(len(X_new)):
        y_list.append(0.2*X_new.X1[x] - 5*X_new.X2[x] + 10*X_new.X3[x])
    y_new = pd.DataFrame(y_list, columns=['Y'])
    df_new = X_new.join(y_new).append(X_og).reset_index().drop(columns=['index'])
    return(df_new)
```

```python
plus_df_X1 = change_features(minus_df, plus_X, 'X1', 0.05, 'plus')
minus_df_X1 = change_features(plus_df, minus_X, 'X1', 0.05, 'minus')
minus_df_X1
```

```python
def cate(y, y_new):
    mean0 = np.mean(y)
    mean1 = np.mean(y_new)
    tao = mean1 - mean0
    return tao
```

```python
minus_cate = {}
for x in minus_X:
    minus_df_new = change_features(minus_df, minus_X, x, 0.05, 'minus')
    c = cate(minus_y, minus_df_new.Y)
    minus_cate[x] = c
pd.DataFrame([minus_cate])
```

```python
plus_cate = {}
for x in plus_X:
    plus_df_new = change_features(minus_df, plus_X, x, 0.05, 'plus')
    c = cate(plus_y, plus_df_new.Y)
    plus_cate[x] = c
pd.DataFrame([plus_cate])
```

### Cancer test data

```python
X = pd.read_csv('cervical_cancer_risks/X.csv')
y = pd.read_csv('cervical_cancer_risks/y.csv')
df = X.join(y)
```

```python
df
```

```python
bin_col = []
for i in X:
    if  len(X[i].unique())<4:
        bin_col.append(i)
```

```python
bin_col
```

```python
for i in range(len(bin_col)):
    print(X[bin_col[i]].value_counts())
```

```python
def change_binary(df, X_feat, alpha):
    # alpha: percentage of values to change
    # Only keep binary variables
    binary = [0,1]
    df = df[df[X_feat].isin(binary)].reset_index()

    df_X = df.iloc[:,:-1]
    # Choose random indices to change values of
    rand = []
    m = int(np.floor(len(df_X)*alpha))
    for i in range(m):
        rand.append(random.randint(0,max(df_X.index)))
    for d in rand:
        df_X[X_feat].loc[d] = np.abs(df_X[X_feat].loc[d] - 1)

    rf = RandomForestClassifier()
    rf.fit(df.iloc[:,:-1], df.Biopsy)
    y_new = pd.DataFrame(rf.predict(df_X), columns=['pred_y'])
    df_new = df_X.join(y_new).reset_index().drop(columns=['index'])
    return(df_new)
```

```python
def change_continuous(df, X_feat, alpha, sig):
    # sig: value to change sigma by
    df_X = df.iloc[:,:-1]
    # Choose random indices to change values of
    std = np.std(df_X[X_feat])
    rand = []
    m = int(np.floor(len(df_X)*alpha))
    for i in range(m):
        rand.append(random.randint(0,max(df_X.index)))
    for d in rand:
        df_X[X_feat].loc[d] = df_X[X_feat].loc[d] + sig * std

    rf = RandomForestClassifier()
    rf.fit(df.iloc[:,:-1], df.Biopsy)
    y_new = pd.DataFrame(rf.predict(df_X), columns=['pred_y'])
    df_new = df_X.join(y_new).reset_index().drop(columns=['index'])
    return(df_new)
```

```python
def cate(y, y_new):
    mean0 = np.mean(y)
    mean1 = np.mean(y_new)
    tao = mean1 - mean0
    return tao
```

```python
X = df.iloc[:,:-1]
cate_values = {}
for X_feat in X:
    if X_feat in bin_col:
        df_new = change_binary(df, X_feat, alpha=0.5)
    else:
        df_new = change_continuous(df, X_feat, alpha=0.5, sig=1)
    c = cate(df.Biopsy, df_new.pred_y)
    cate_values[X_feat] = c
pd.DataFrame([cate_values])
```

```python

```


```python
from sympy import init_session
init_session()
```

    IPython console for SymPy 1.5.1 (Python 3.6.9-64-bit) (ground types: gmpy)

    These commands were executed:
    >>> from __future__ import division
    >>> from sympy import *
    >>> x, y, z, t = symbols('x y z t')
    >>> k, m, n = symbols('k m n', integer=True)
    >>> f, g, h = symbols('f g h', cls=Function)
    >>> init_printing()

    Documentation can be found at https://docs.sympy.org/1.5.1/

```python
#init_printing?
```

### Parameters and two Gaussians

```python
rc, a, a1, a2 = symbols('rc, a, a1 a2', positive=True, real=True)
```

```python
u1=x**2*exp(-a1*x**2)
u2=x**2*exp(-a2*x**2)
gp=x**4*exp(-a*x**2)
w=(x-rc)**2
u1, u2, gp, w
```

### Radial wavefunction $u$
We don't solve the radial equation for $R(r)$, but rather for $u(r) = r \cdot R(r)$.
So, if $R$ is a s-function, $u$ is a p-function, and if $R$ is a p-function, $u$ is a d-function, etc.

$S = \int r^2dr R^2 = \int dr\, u^2$

$T = \int r^2dr R_1 \hat{T} R_2 = \int dr\, u_1\, \frac{d^2}{dr^2} \, u_2 $

### Normalization constant

```python
N=integrate(u1*u1, (x, 0, oo))
N
```

```python
1/sqrt(N)
```

```python
printing.sstrrepr(1/sqrt(N))
```

### CAP

```python
W=integrate(u1*w*u2, (x, rc, oo))
W
```

```python
Ws=W.simplify()
Ws
```

```python
Ws.factor()
```

```python
printing.sstrrepr(Ws)
```

    '-(6*sqrt(pi)*a1*rc**2*sqrt(a1 + a2)*exp(rc**2*(a1 + a2))*erf(rc*sqrt(a1 + a2)) - 6*sqrt(pi)*a1*rc**2*sqrt(a1 + a2)*exp(rc**2*(a1 + a2)) + 2*a1*rc + 6*sqrt(pi)*a2*rc**2*sqrt(a1 + a2)*exp(rc**2*(a1 + a2))*erf(rc*sqrt(a1 + a2)) - 6*sqrt(pi)*a2*rc**2*sqrt(a1 + a2)*exp(rc**2*(a1 + a2)) + 2*a2*rc + 15*sqrt(pi)*sqrt(a1 + a2)*exp(rc**2*(a1 + a2))*erf(rc*sqrt(a1 + a2)) - 15*sqrt(pi)*sqrt(a1 + a2)*exp(rc**2*(a1 + a2)))*exp(-rc**2*(a1 + a2))/(16*(a1**4 + 4*a1**3*a2 + 6*a1**2*a2**2 + 4*a1*a2**3 + a2**4))'

```python
W=integrate(gp*w, (x, rc, oo))
W
```

```python
Wf=W.factor()
Wf
```

```python
Wf.simplify()
```

```python
printing.sstrrepr(Wf.simplify())
```

    '-rc*exp(-a*rc**2)/(8*a**3) - 3*sqrt(pi)*rc**2*erf(sqrt(a)*rc)/(8*a**(5/2)) + 3*sqrt(pi)*rc**2/(8*a**(5/2)) - 15*sqrt(pi)*erf(sqrt(a)*rc)/(16*a**(7/2)) + 15*sqrt(pi)/(16*a**(7/2))'

```python

```


```python
# G2: Standard 2D Model

# Make SymPy available to this program:
import sympy
from sympy import *

# Make GAlgebra available to this program:
from galgebra.ga import *
from galgebra.mv import *
from galgebra.printer import Fmt, GaPrinter, Format
    # Fmt:       sets the way that a multivector's basis expansion is output.
    # GaPrinter: makes GA output a little more readable.
    # Format:    turns on latex printer.
from galgebra.gprinter import gFormat, gprint
gFormat()
```

```python
# g2: The geometric algebra G^2.
g2coords = (x,y) = symbols('x y', real=True)
g2 = Ga('e', g=[1,1,1], coords=g2coords)
(ex, ey) = g2.mv()

grad = g2.grad
from galgebra.dop import *
pdx = Pdop(x)
pdy = Pdop(y)
```

```python

```


# Übungsblatt 4: MCMC

 * [Aufgabe 1](#Aufgabe-1)
 * [Aufgabe 2](#Aufgabe-2)

---

```python

```

---
## Aufgabe 1
Erzeugen Sie Paare von Zufallszahlen (x, y) die gemäß der zweidimensionalen Dichte

\begin{equation}
    f(x, y) \propto \exp\left(-((x-2)x + y^2)/10\right)\cdot\sin^2(x\cdot y + x)
\end{equation}

verteilt sind. Bestimmen Sie daraus die Verteilung $\rho(r)$ mit $r^2 = x^2 + y^2$.

---

```python

```

---
## Aufgabe 2
Ein Experiment beobachtet $n$ Ereignisse. Die Nachweiswahrscheinlichkeit für ein Ereignis sei $ε$. Die wahre Anzahl ist Poisson-verteilt mit einem Mittelwert $μ$, der eine Funktion zweier Theorieparameter $a$ und $b$ ist, $μ = a^b$. Aus Kalibrationsmessungen ist bekannt dass $ε$ Gauss-verteilt ist mit $ε = 0.75 \pm 0.05$. Vom Parameter $b$ ist bekannt dass er nahe bei $b = 1$ liegt. Für ihn wird eine gleichförmige Verteilung im Bereich $0.9 < b < 1.1$ angenommen. Der Parameter $a$ muss positiv sein. Verwenden Sie bayes’sche Statistik und MCMC, und bestimmen Sie für $n = 1$ und $n = 10$ die posterior Verteilung von $a$ nach Integration über die Nuissance-Parameter $ε$ und $b$.

---

```python

```


# Electronics Worked Example

This notebook starts to explore a worked example of how we can use a package like `lcapy` to support the production of technical eductional materials in a way that guarantees correctness and provides opportunities to developing self-text activities.

The aim is to be able to generate a set of materials where all the equations and all the calculations are rendered mechanically from a original circuit description. This would then guarantee their correctness relative to the original circuit.

From an original circuit description, we can generate:

- a rendered circuit diagram;
- symbolic expressions derived from simple automated analysis of the circuit;
- automatically generated problem solutions for calculations based around particular component values and voltages etc.

## `lcapy`

As described elsewhere, `lcapy` is a linear circuit analysis package that can be used to describe, display and analyse the behaviour of a wide range of linear analogue electrical circuits.

`lcapy` use a circuit description that can be used to generate an circuit diagram as the basis for a wide range of analyses.

```python
%%capture
try:
    import lcapy
except:
    %pip install lcapy

#  https://github.com/dawbarton/pdf2svg
#!brew install pdf2svg
```

Circuits are defined by generating a graph that connects numbered nodes with edges representing a particular component. Layout instructions describe the orientation of the edge, and labels can be added as required.

In actual use, it would make sense to build up a library of comment circuit blocks that can be visually reviewed, and then the corresponding definition copied and reused, with modification as required.

```python
from lcapy import Circuit

cct = Circuit("""
V1 1 0; down
R1 1 2; left=2, i=I_1, v=V_{R_1}
R2 1 3; right=2, i=I_2, v=V_{R_2}
L1 2 0_1; down, i=I_1, v=V_{L_1}
L2 3 0_3; down, i=I_1, v=V_{L_2}
W 0 0_3; right
W 0 0_1; left""")

# Render the circuit
cct.draw(scale=3)

# We can also save a generated circuit diagram to an image file
#cct.draw(scale=3, filename='circuit_diagram.svg')
```

## Example Production Workflow

To provide an example of how we might use `lcapy` consider the following teaching example taken from *The following section is a reworking of http://www.open.edu/openlearn/science-maths-technology/introduction-electronics/content-section-3.1 .*

### Voltage Divider

Voltage dividers are widely used in electronic circuits to create a reference voltage, or to reduce the amplitude of a signal. The figure below shows a voltage divider. The value of $V_{out}$ can be calculated from the values of $V_S$, $R_1$ and $R_2$.

```python
import lcapy
from lcapy import Circuit

#We can create a schematic for the voltage divider using lcapy
#This has the advantage that circuit description is also a model
#The model can be analysed and used to calculate voltages and currents, for example,
#  across components if component values and the source voltage are defined

#Figure:  A voltage divider circuit
sch='''
VS 1 0 ; down
W 1 2 ;  right, size=2
R1 2 3 ; down
R2 3 4; down
W 3 5; right
P1 5 6; down,v=V_{out}
W 4 6; right
W 4 0; left
'''

#Demonstrate thate we can write the descriptioon to a file
fn="voltageDivider.sch"
with open(fn, "w") as text_file:
    text_file.write(sch)

#  and then create the circuit model from the (persisted) file
cct = Circuit(fn)
```

The schema is also retrievable from the circuit object:

```python
cct.sch
```

    VS 1 0 ; down
    W 1 2 ;  right, size=2
    R1 2 3 ; down
    R2 3 4; down
    W 3 5; right
    P1 5 6; down,v=V_{out}
    W 4 6; right
    W 4 0; left

```python
#Draw the circuit diagram that corresponds to the schematic description
cct.draw(style='american', draw_nodes=False, label_nodes=False) #american, british, european
#Draw function is defined in https://github.com/mph-/lcapy/blob/master/lcapy/schematic.py
#The styles need tweaking to suit OU convention - this requires a minor patch to lcapy
#Styles defined in https://github.com/mph-/lcapy/blob/master/lcapy/schematic.py#Schematic.tikz_draw
```

In the first instance, let’s assume that  is not connected to anything (for voltage dividers it is always assumed that negligible current flows through ). This means that, according to Kirchhoff’s first law, the current flowing through  is the same as the current flowing through . Ohm’s law allows you to calculate the current through $R_1$. It is the potential difference across that resistor, divided by its resistance. Since the voltage  is distributed over two resistors, the potential drop over $R_1$ is $V_{R_1}=V_S - V_{out}$.

The equation at the end of the last paragraph is written explicitly as LateX.

But we can also analyse the circuit using `lcapy` to see what the equation *should* be by deriving it automatically:

The voltage across $R_2$, $V_{out}$, is given as:

```python
cct.R1.v

#We can't do anything about the order of the variables in the output expression, unfortunately
#It would be neater if sympy sorted fractional terms last but it doesn't...
```

We can get an expression for the output voltage, $V_{out}$, or its calculated value in a couple of ways:

```{note}
The circuit's node numbers can be displayed on the schematic if required: simply set `label_nodes=True` in the `draw()` statement.
```

For example, we can find the voltage across the appropriately numbered nodes by getting the open circuit voltage across nodes 3 and 4 (that is, across R2).

```python
cct.Voc(3,4)['t']
```

The output voltage can also be obtained by direct reference to the appropriate component:

```python
cct.R2.v
```

We can create a reference for our voltages as symbolic elements.

For example, we can get an expression for $V_{out}$:

```python
#sympy is a symbolic maths package
from sympy import Symbol, Eq

#If we add .expr to the voltages, we can get the sympy representation of voltage and current equations
# that are automatically derived from the model.
vout_expr = cct.R2.v.expr

vout_expr
```

We can also reference this value via a string:

```python
out_component = 'R2'

cct[out_component].v.expr
```

```python
cct.R2.relname
```

    'R2'

And an expression for the voltage across $R_1$:

```python
v_r1_expr = cct.R1.v.expr

v_r1_expr
```

Working with sympy symbols, we can perform a substitution if expressions match exactly.

In this case, we can swap in $V_{out}$ for the expression returned from the analysis to give us an expression in the form we want for $R_1$:

```python
vout = Symbol('V_out')

v_r1_expr.subs(vout_expr, vout)
```

We can now create an equation to declare the relationship between a symbol for $R_1$ and the expression that defines its value:

```python
#I don't know how to get the symbols from the circuit as sympy symbols
# so create them explicitly

v_r1 = Symbol("V_{R_1}")

# Define the equation
Eq( v_r1, v_r1_expr.subs(vout_expr, vout) )
```

```{note}
In a traditional set of teaching materials, we might work through the derivations of various formulae describing how to calculate particulalr quantities.

In order to render symbols correctly, we might author our document using LaTeX. Jupyter notebooks are quite comfortable with rendering LaTeX inline, or as part of a multiline block, so let's create an example of the sort of proof we might see in a traditionally authored text.

*The following expressions are hand written using LaTeX*
```

The current through $R_1$ ($I_{R_1}$) is given by $I_{R_1}=\displaystyle\frac{(V_S-V_{out})}{R_1}$

Similarly, the current through $R_2$ is given by $I_{R_2}=\displaystyle\frac{V_{out}}{R_2}$

Kirchoff’s first law tells you that $I_{R_1}=I_{R_2}=$, and therefore

$\displaystyle\frac{V_{out}}{V_{R_2}}=\frac{(V_S-V_{out})}{R_1}$

Multiplying both sides by $R_1$ and by $R_2$ gives

$R_1V_{out}=R_2(V_S-V_{out})$

Then multiplying out the brackets on the right-hand side gives

$R_1V_{out}=R_2V_S-R_2V_{out}$

This can be rearranged to

$R_1V_{out}+R_2V_{out}=R_2V_S$

giving

$(R_1+R_2)V_{out}=R_2V_S$

and therefore the fundamental result is obtained:

$V_{out}=\displaystyle\frac{R_2V_S}{(R_1+R_2)}$

```{note}
One thing we might note at this point is that by authoring in the proof in the same document as the circuit diagram, we reduce the "distance" between the circuit diagram and the derivation that purports to relate to it but there is still the potential for error.

The distance between the image and the derivation also means that if we change the labeling on the diagram, for example, the derivation will no longer properly describe the components as labeled in the diagram.
```

From our symbolic expressions, we can render an equation for $V_{out}$ very straightforwardly:

```python
#We can find this from quantities we have derived through analysis of the presented circuit
Eq(vout, vout_expr)
```

Recall that the expression `vout_expr` was calculated automatically from the original circuit representation, which was also used to render the circuit diagram. The expression is guaranteed to be correct for the circuit we have defined, although it may not be presented in the order of the form we would like it to be in.

As well as accessing the voltage across a component, we can also access and expression for the current flowing through a component.

For example, let's get the expression for the current flowing through $R_1$ and assign it to an appropriate symbol:

```python
Eq(Symbol('I_{R_1}'), cct.R1.i.expr)

# The following equation is generated by the symbolic analysis...
```

```{note}
There is still distance in each of these uses of `Eq()`, where we "casually" state the equivalence of left and right hand components. Could we derive the symbol for the left hand side automatically, somehow?
```

We can also get an expression for the current through $R_2$:

```python
#We get the following from the circuit analysis, as above...
cct.R2.i.expr

# We note that the circuit analysis returns equal expressions for I_R_1 and I_R_2
# which gives some sort of reinforcement to the idea of Kirchoff's Law...

# The following equation is generated by the symbolic analysis...
```

Being able to render expressions that describe the values of particular quantities in algebraic terms means we may be able to create derivations that are "necessarily" true.

But we can do more than that, and also start to insert numbers into the expressions.

Consider the following exercise:

#### Exercise

Suppose  $V_S= 24 V$ and $R_2 = 100\Omega$. You want $V_{out} = 6 V$. What value of $R_1$ do you need?

#### Answer

```{note}
In a traditional set of materials, we are likely to write out the steps to the solution manually (a process which is subject to error and likely to need checking for correctness as well as sense) and then manually substitute in the required values and perform the calculation (which requires more checking).
```

Rearranging the equation for $V_{out}$ gives

$V_{out}(R_1+R_2)=R_2V_S$

and therefore:

$(R_1+R_2)=\displaystyle\frac{R_2V_S}{V_{out}}$

which means the equation for $R_1$ is:

$R_1=\displaystyle\frac{R_2V_S}{V_{out}}-R_2$

Substituting in the values given and performing the calculation gives us the required solution:

$R_1=\displaystyle\frac{100\Omega \times 24V}{6V}-100\Omega = 400\Omega-100\Omega=300\Omega$

```{note}
Once again, all the above steps are manuually derived and presented using hand crafted LaTex.
```

How might we set about creating an equivalent solution, but guaranteed to be correct?

We essentially want to solve for $R_1$ in the following expression (which was automatically derived by analysis of the presented circuit representation):

```python
Eq(vout, vout_expr)
```

The steps we need to take are a simplification of the expression, with value substitution:

```python
from sympy import sympify

#This is clunky - is there a proper way of substituting values into lcapy expressions?
Eq(6, sympify(str(vout_expr)).subs([('VS', 24), ('R2', 100)]))
```

Rearranging, we need to solve the following for $R_1$, generated the output expression automatically through symbolic analysis:

```python
Eq(vout_expr - vout, 0)
```

`sympy` can solve such equations for us directly, so let's see how it solves things symbolically for $R_1$:

```python
from sympy import solve

Eq(Symbol('R1'),solve(sympify(str(vout_expr-vout)),'R1')[0])

#The following equation is generated by the symbolic analysis...
```

To solve the equation numerically, we can substitute values into the `sympy` expression as follows:

```python
solve(sympify(str(vout_expr-vout)).subs([('VS',24), ('R2',100),('V_out',6)]),'R1')[0]
```

We can craft an equation to display the solution for us:

```python
VALUES = [('VS', 24),
          ('R2', 100),
          ('V_out', 6)]

SOLVE_FOR = 'R1'

Eq(Symbol(SOLVE_FOR), solve(sympify(str(vout_expr-vout)), SOLVE_FOR)[0].subs(VALUES))

#The following result is calculated by the symbolic analysis...
```

A key point about this is that we can script in different component values and display the correct output.

We can also do partial solutions which might be useful if we want to build up the steps to a solution, with partial substitution of values along the way:

```python
Vs=20; Vout=5
R1=solve(sympify(str(vout_expr-vout)),'R1')[0].subs([('VS',Vs),('V_out',Vout)])

print('For V_S={Vs}V and V_out={Vout}V, we need R1={R1}.'.format(Vs=Vs,Vout=Vout,R1=R1))
```

    For V_S=20V and V_out=5V, we need R1=3*R2.

Alternatively, we can create a function to solve for any single missing value.

The following will calculate the relevant solution:

```python
def soln(cct, out_component=None, values=None):
    if values is None:
        values={'VS':24, 'R1':'', 'R2':100, 'V_out':6}
    outval=[v for v in values if not values[v]]
    invals=[(v, values[v]) for v in values if values[v] ]
    if len(outval)!=1 or len(invals)!=3:
        return 'oops'

    outval = outval[0]
    out_component = out_component if out_component else outval
    vout_expr = cct[out_component].v.expr
    print(vout_expr)
    print(invals)
    return 'Value of {} is {}'.format(outval,
                                      solve(sympify(str(vout_expr-vout)).subs(invals), outval)[0])

```

For example, using the default values baked into the function:

```python
soln(cct, 'R2')
```

    R2*VS/(R1 + R2)
    [('VS', 24), ('R2', 100), ('V_out', 6)]

    'Value of R1 is 300'

Or using values we pass in:

```python
soln(cct, 'R2', {'VS':24, 'R2':'', 'R1':300, 'V_out':6})
```

    R2*VS/(R1 + R2)
    [('VS', 24), ('R1', 300), ('V_out', 6)]

    'Value of R2 is 100'

With these pieces in place, we can also consider how we might automatically generate circuit diagrams with specified component values, and then perform, or check, calculations against them.

For example, the following function wraps a specific circuit definition and allows specific component values to be passed in:

```python
#We can also explore a simple thing to check the value from a circuit analysis
def cct1(V='24',R1='100',R2='100'):
    R1 = '' if R1 and float(R1) <=0 else R1
    sch='''
    VS 1 0 {V}; down
    W 1 2 ;  right, size=2
    R1 2 3 {R1}; down
    R2 3 4 {R2}; down
    W 3 5; right, size=2
    P1 5 6; down,v=V_{{out}}
    W 4 6; right, size=2
    W 4 0; left
    '''.format(V=V,R1=R1,R2=R2)

    cct = Circuit()
    cct.add(sch)
    cct.draw(label_nodes=False)

    #The output voltage, V_out is the voltage across R2
    txt='The output voltage, $V_{{out}}$ across $R_2$ is {}V.'.format(cct.R2.v if R1 else V)
    return cct
```

We can render the circuit simply by calling the function:

```python
_ = cct1()
```

It's trivial to make an interactive widget built around the previous function that will create a diagram for us with components labeled according to values specified by input sliders.

```{note}
In a live Jupyter backed UI, the following example will render several interactive widgets that allow component values to be specified; once selected, the circuit will be drawn and a missing value calculated automatically.
```

```python
from ipywidgets import interact_manual
cct_w = None

@interact_manual
def i_cct1(V='24', R1='', R2='100'):
    global cct_w
    cct_w = cct1(V=V,R1=R1,R2=R2)

#We could then select R and V values and extend the function
# to calculate V_out automatically
```

    interactive(children=(Text(value='24', description='V'), Text(value='', description='R1'), Text(value='100', d…

```python
i_cct1()
```

```python
# We could also plot V_out vs R_1 for given V_S and R_2?
```

```python
cct_w.sch
```

    VS 1 0 24; down
    W 1 2 ;  right, size=2
    R1 2 3 ; down
    R2 3 4 100; down
    W 3 5; right, size=2
    P1 5 6; down,v=V_{out}
    W 4 6; right, size=2
    W 4 0; left

```python

```


# 12 ODE Applications (Projectile motion)
Let's apply our ODE solvers to some problems involving balls and projectiles.

The `integrators.py` file from [Lesson 10](https://asu-compmethodsphysics-phy494.github.io/ASU-PHY494/2019/02/19/10_ODEs/) is used here (and named [`ode.py`](https://github.com/ASU-CompMethodsPhysics-PHY494/PHY494-resources/tree/master/12_ODE_applications/ode.py)).

## Contents

1. Projectile with linear air-resistance (theory and full code)
2. Baseball physics (theory), see [baseball_solution](baseball_solution.ipynb) for the notebook

```python
import numpy as np
import ode
```

```python
%matplotlib inline
import matplotlib.pyplot as plt
plt.matplotlib.style.use('ggplot')
```

## Projectile with linear air-resistance
Linear drag force

$$
\mathbf{F}_1 = -b_1 \mathbf{v}
$$

Equations of motion with force due to gravity $\mathbf{g} = -g \hat{\mathbf{e}}_y$

\begin{align}
\frac{d\mathbf{r}}{dt} &= \mathbf{v}\\
\frac{d\mathbf{v}}{dt} &= - g \hat{\mathbf{e}}_y -\frac{b_1}{m} \mathbf{v}
\end{align}

Bring into standard ODE form for

$$
\frac{d\mathbf{y}}{dt} = \mathbf{f}(t, \mathbf{y})
$$

as

$$
\mathbf{y} = \begin{pmatrix}
x\\
y\\
v_x\\
v_y
\end{pmatrix}, \quad
\mathbf{f} = \begin{pmatrix}
v_x\\
v_y\\
-\frac{b_1}{m} v_x\\
-g -\frac{b_1}{m} v_y
\end{pmatrix}
$$

(Based on Wang 2016, Ch 3.3.1)

```python
def simulate(v0, h=0.01, b1=0.2, g=9.81, m=0.5):

    def f(t, y):
        # y = [x, y, vx, vy]
        return np.array([y[2], y[3], -b1/m * y[2], -g - b1/m * y[3]])

    vx, vy = v0
    t = 0
    positions = []
    y = np.array([0, 0, vx, vy], dtype=np.float64)

    while y[1] >= 0:
        positions.append([t, y[0], y[1]])  # record t, x and y
        y[:] = ode.rk4(y, f, t, h)
        t += h

    return np.array(positions)

def initial_v(v, theta):
    x = np.deg2rad(theta)
    return v * np.array([np.cos(x), np.sin(x)])
```

```python
r = simulate(initial_v(200, 30), h=0.01, b1=1)
```

```python
plt.plot(r[:, 1], r[:, 2])
plt.xlabel(r"distance $x$ (m)")
plt.ylabel(r"height $y$ (m)");
```

```python
for angle in (5, 7.5, 10, 20, 30, 45):
    r = simulate(initial_v(200, angle), h=0.01, b1=1)
    plt.plot(r[:, 1], r[:, 2], label=r"$\theta = {}^\circ$".format(angle))
plt.legend(loc="best")
plt.xlabel(r"distance $x$ (m)")
plt.ylabel(r"height $y$ (m)");
```

## Simple Baseball physics

- quadratic air resistance (with velocity-dependent drag coefficient)
- Magnus force due to spin

### Quadratic air resistance
Occurs at high Reynolds numbers, i.e., turbulent flow. Only approximate:

$$
\mathbf{F}_2 = -b_2 v \mathbf{v}
$$

### Magnus effect

**Magnus effect**: airflow is changed around a spinning object. The Magnus force is

$$
\mathbf{F}_M = \alpha \boldsymbol{\omega} \times \mathbf{v}
$$

where $\boldsymbol{\omega}$ is the ball's angular velocity in rad/s (e.g., 200/s for a baseball).

For a sphere the proportionality constant $\alpha$ can be written

$$
\mathbf{F}_M = \frac{1}{2} C_L \rho A \frac{v}{\omega} \boldsymbol{\omega} \times \mathbf{v}
$$

where $C_L$ is the lift coefficient, $\rho$ the air density, $A$ the ball's cross section. (Advantage of defining $C_L$ this way: when spin and velocity are perpendicular, the Magnus force is simply $F_M = \frac{1}{2} C_L \rho A v^2$.)

$C_L$ is mainly a function of the *spin parameter*

$$
S = \frac{r\omega}{v}
$$

with the radius $r$ of the ball. In general we write

$$
\mathbf{F}_M = \frac{1}{2} C_L  \frac{\rho A r}{S} \boldsymbol{\omega} \times \mathbf{v}
$$

For a baseball, experimental data show approximately a power law dependence of $C_L$ on $S$

$$
C_L = 0.62 \times S^{0.7}
$$

All together:

\begin{align}
\mathbf{F}_M &= \alpha\ \boldsymbol{\omega} \times \mathbf{v}\\
v &= \sqrt{\mathbf{v}\cdot\mathbf{v}}\\
S &= \frac{r\omega}{v}\\
C_L &= 0.62 \times S^{0.7}\\
\alpha &= \frac{1}{2} C_L  \frac{\rho A r}{S}
\end{align}

### Equations of motion

\begin{align}
\frac{d\mathbf{r}}{dt} &= \mathbf{v}\\
\frac{d\mathbf{v}}{dt} &= -g \hat{\mathbf{e}}_y \mathbf{v} -\frac{b_2}{m} v \mathbf{v} + \alpha\ \boldsymbol{\omega} \times \mathbf{v}\\
\end{align}

(quadratic drag $-\frac{b_2}{m} v \mathbf{v}$ included.)

### Baseball simulation

Implement the full baseball equations of motions:
- gravity $a_\text{gravity}$
- quadratic drag $a_\text{drag}$
- Magnus effect $a_\text{Magnus}$

For the cross product you can look at [numpy.cross()](https://docs.scipy.org/doc/numpy/reference/generated/numpy.cross.html).

We will live-code the baseball simulation in class (i.e., build it from scratch), but if you want to work on this problem on your own and need some starter code, see below.

#### Baseball  code
Full solution will be posted as [`baseball_solution.ipynb`](baseball_solution.ipynb).

#### Simulate throws

Simulate baseball throw for initial velocity $\mathbf{v} = (30\,\text{m/s}, 0)$.

Plot x vs y and x vs z (to see curving).

Try out different spins; a good value is $\boldsymbol{\omega} = 200\,\text{rad/s} \times (0, 1, 1)$.

Simulate the baseball throw with
- almost no spin: $\omega = 0.001 \times (0, 0, 1)$  (our code does not handle $\omega = 0$ gracefully...)
- full upward spin: $\omega = 200 \times (0, 0, 1)$
- sideways spin: $\omega = 200 \times (0, 1, 1)$

#### Plotting

Plot the three scenarios in 2D planes: x-y (side view) and x-z (top view).

#### 3D plot
Use simple `matplotlib` 3D plot. (BONUS: Make it work with vpython)

If we use the [`ipympl` backend for matplotlib](http://ipython.readthedocs.io/en/stable/interactive/plotting.html) then we will be able to interactively rotate our [matplotlib 3D graphics](http://matplotlib.org/mpl_toolkits/mplot3d/tutorial.html). (Note: If this does not seem to work, disable adblockers and allow javascript on the page.)

```python
%matplotlib ipympl
```

```python
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
ax.plot(r[:,1], r[:,3], r[:,2], 'o', label="no spin")
# ...

# hand of the catcher, 0.2m above homeplate
ax.plot([18.4, 18.4], [0, 0], [0, 0.2], color="black", lw=6)

ax.set_xlabel("$x$ (m)")
ax.set_ylabel("$z$ (m)")
ax.set_zlabel("$y$ (m)")
ax.legend(loc="upper left", numpoints=1)
ax.figure.tight_layout()
```

## Reynolds number

$$
\text{Re} = \frac{\rho v L}{\mu}\\
\text{Re} > 2300\quad\text{flow turbulent}
$$

* density $\rho$: air  1.275 kg/m^3 (kilograms per cubic meter)
* fluid viscosity $\mu$: air 1.845×10^-5 Pa s (pascal seconds) (at 25 °C)

(from Wolfram Alpha)

```python
rho_air = 1.275    # kg/m^3
mu_air = 1.845e-5  # Pa s
```

```python
L = 0.05  # m
v = 200   # m/s
```

```python
def ReynoldsNumber(v, L, rho=rho_air, mu=mu_air):
    return rho*v*L/mu
```

```python
ReynoldsNumber(v, L)
```

    691056.9105691056

This means that we really should be using quadratic air resistance for the projectile.

```python

```


# Homework 8: Introduction to Numerical ODE Solutions (Extra Credit)
*Source: Eric Ayars, PHYS 312 @ CSU Chico*

This is an OPTIONAL homework assignment that you can complete for extra credit.

Please complete this homework assignment in code cells in the iPython notebook. Include comments in your code when necessary. Please rename the notebook as SIS ID_HW08.ipynb (your student ID number) and save the notebook once you have executed it as a PDF (note, that when saving as PDF you don't want to use the option with latex because it crashes, but rather the one to save it directly as a PDF).

The homework should be submitted on bCourses under the Assignments tab (both the .ipynb and .pdf files). Please label your submission with your student ID number (SIS ID)

This is often used as a workshop in the course, however, you'll only get graded on your solutions to the three problem. However, I've left the background background material in place, because it will be helpful in completing those problems.

```python
# Run this cell before preceding
%matplotlib inline
import numpy as np
import matplotlib.pyplot as plt
```

## Ordinary Differential Equation (ODE)

An ordinary differential equation is an equation that takes the following form:

$$F(t,x,x',x'',\dots) = 0$$

where $x$ is a function of $t$ and the $'$ symbol denotes derivatives:

$$x' = \frac{dx}{dt}$$
$$x'' = \frac{d^2x}{dt^2}$$
$$\vdots$$

An example is

$$x' + x = 0$$

To solve such an equation, we need to specify an *initial condition*: a set of values $(t_0, x_0)$ that our solution must pass through. This is because there are multiple solutions which can satisfy that equation. Any solution of the form

$$x(t) = Ae^{-t}$$

satisfies the differential equation above. So by requiring that this curve pass through a particular point $(t_0, x_0)$, we can determine $A$:

$$A  = \frac{x_0}{e^{-t_0}}$$

Another way to visualize is this is with the aid of a "slope field": a plot that, for various points $(t,x)$ shows what $x(t)$ must look like locally by evaluating the derivative $x'$ at that point:

```python
# Initial condition
t0 = 0.0
x0 = 0.75

# Make a grid of x,t values
t_values = np.linspace(t0, t0+3, 20)
x_values = np.linspace(-np.abs(x0)*1.2, np.abs(x0)*1.2, 20)
t, x = np.meshgrid(t_values, x_values)

# Evaluate derivative at each x point
xdot = -x

plt.figure()

# Plot slope field arrows
plt.quiver(t,x, np.ones(t.shape), xdot,color='b')

# Plot solution
A = x0 / np.exp(-t0)
plt.plot(t_values,A * np.exp(-t_values),color='r')

# Plot initial condition
plt.plot(t0,x0,'go',markersize=8)

plt.xlabel('t')
plt.ylabel('x(t)')
plt.title("Slope field and a solution of $x'=x$")
plt.show()
```

With those two pieces of information--the differential equation and an initial condition--we are able to write down a closed-form solution $x(t)$. But for a general differential equation, even if you have an initial condition, it is difficult to write down $x(t)$ in closed form. For example, if the equation is nonlinear or, if you have a set of *coupled* differential equations, as we frequently encounter in physics, numerical methods are indispensable.

## Euler method

### Definition of the Euler method

Suppose we have the differential equation

$$\frac{dx}{dt} = f(x,t)$$

This means that, given a point in the system $(x_0, t_0)$, we have a way to compute the derivative $dx/dt$ at that point. But it may be difficult or impossible to analytically integrate $f(x,t)$ to find a closed form for $x(t)$. Instead, we rely on numerical methods to estimate solutions. More specifically, given an *initial condition* $(x_0, t_0)$, where $x(t_0) = x_0$, the goal is to find a numerical method to calculate $x(t)$ for $t > t_0$.

The most basic Euler method is based on the simple observation that

$$x(t+\Delta t) = x(t) + \int_{t}^{t+\Delta t} \left(\frac{dx}{dt}\right) dt =  x(t) + \int_{t}^{t+\Delta t} f(x(t),t) dt $$

If we cannot explicitly take that integral, but we have a way to calculate $f(x,t)$, then the first thing we would try is

$$x(t+\Delta t) \approx x(t) + f(x(t), t) \cdot \Delta t$$

Now let us try to make this into code. Suppose we have a list of times $\{t_i\}$ such that $t_{i+1} - t_i = \Delta t$ (generated by `np.arange` or `np.linspace`, for example). Then given $x_0$ at $t_0$, we calculate $x_i$ according to the rule

$$x_i = x_{i-1} + f(x_{i-1},t_{i-1})\Delta t$$

So as long as we can write the first derivative in the form above, we have a way to attack this problem.

For example, we can numerically solve a problem like

$$v' = 1-v^2$$
$$\rightarrow v_i = v_{i-1} + f(v_{i-1}, t_{i-1}) \Delta t = v_{i-1} + (1-v_{i-1}^2)\Delta t$$
Given an initial $(t_0, v_0)$, we can use the Euler method to solve this equation, which describes the velocity (denoted $v$ here) of a particle falling but experiencing a drag force (see lecture). We know that the solution of such an equation should be that the velocity of the particle should increase quickly at first (due to constant gravitational force) but then asymptote to some terminal value because the drag force increases with velocity. Let's see this:

```python
# Basic example of Euler method
t_0 = 0.0 # initial time condition
v_0 = 0.0 # initial velocity condition

# Generate some times t_i
t_data = np.linspace(0,100,1000)

# Placeholder array for velocities v_i
v_data = np.zeros(1000)

v_data[0] = v_0
N = len(t_data)

# use Euler method to estimate v_i for each i
for i in range(1,N):

    f = 1 - v_data[i-1]**2 # f(v_{i-1})
    dt = t_data[i] - t_data[i-1] # time interval
    v_data[i] = v_data[i-1] + f * dt # calculate v_i

# Plot results
plt.figure()
plt.plot(t_data, v_data)
plt.xlabel("Time")
plt.ylabel("Velocity")
plt.title("Velocity of particle falling and experiencing drag")
plt.show()
```

So we have established a technique to approximate solutions to some first-order differential equations. Note that these solutions still have some error--flip back to the workshop on integration techniques to remind yourself of this.

At first, being able to solve only first-order differential equations seems very restrictive. But actually, it is enough for us to start modeling real systems and observing interesting behaviors. First, let us try to convert a second order differential equation, such as Newton's second law, into a set of first order differential equations, which we now know how to solve.

### Example: Euler Method and a Second-Order ODE

Commonly we have a set of *second-order* differential equations. For example, the harmonic oscillator takes this form:

$$F = ma = m\frac{d^2x}{dt^2} = -kx$$
$$\rightarrow \frac{d^2x}{dt^2} + \frac{k}{m}x = 0$$
But we can rewrite as a set of first-order differential equations by noting that

$$a = \frac{dv}{dt}$$

and

$$v = \frac{dx}{dt}$$

So the force equation above becomes a pair of equations:

\begin{align}
x' &= \frac{dx}{dt} =  v \\
v' &= \frac{dv}{dt} = -\frac{k}{m}x
\end{align}

This means that to form a solution, we need three numbers for the initial condition, $(t_0, x_0, v_0)$ where $x(t_0) = x_0$ and $v(t_0) = v_0$. As we did above, let us write this down in terms of the values $x_i$, $v_i$, and $t_i$:

\begin{align}
x_i &= x_{i-1} + v_{i-1} \Delta t \\
v_i &= v_{i-1} + \left(-\frac{k}{m}x_{i-1}\right)\Delta t
\end{align}

In the examples below, I will continue to take $t_0 = 0$

```python
# Use Euler method to solve coupled first order ODE
import numpy as np
%matplotlib inline
import matplotlib.pyplot as plt

km = 0.3 # value of k / m

# Initial conditions
x_0 = 1.0
v_0 = 0.0

# Number of timesteps
T = 1000
dt = 0.1 #size of time step (Delta t)

def Euler(t0, x0, v0, T, dt):
    x_data = np.zeros(T)
    v_data = np.zeros(T)

    t_data = np.arange(T) * dt + t0
    x_data[0] = x0
    v_data[0] = v0

    for i in range(1,T):
        x_data[i] = x_data[i-1] + v_data[i-1] * dt
        v_data[i] = v_data[i-1] + (-km * x_data[i-1]) * dt

    return t_data, x_data, v_data

t_data, x_data, v_data = Euler(0.0, x_0, v_0, T, dt)

# Analytical solutions for (x(t), v(t)) assuming x_0 = 1.0, v_0 = 0, t_0 = 0
analytical_x = np.cos(np.sqrt(km)*t_data)
analytical_v = -np.sqrt(km)*np.sin(np.sqrt(km)*t_data)

plt.figure(figsize=(8,8))
plt.subplot(211)
plt.plot(t_data, x_data, label="numerical")
plt.plot(t_data, analytical_x,label="analytical")
plt.ylabel("Position")
plt.legend()
plt.title("Position of the mass on spring")

plt.subplot(212)
plt.plot(t_data, v_data, label="numerical")
plt.plot(t_data, analytical_v, label="analytical")
plt.ylabel("Velocity")
plt.xlabel("Time")
plt.legend()
plt.title("Velocity of the mass on spring")

# Plot error in position as a function of time
plt.figure()
plt.plot(t_data, np.abs(x_data - analytical_x))
plt.ylabel("$|x_i - x_{analytical}|$")
plt.xlabel("Time")
plt.title("Absolute error in position")
plt.show()
```

### Problem 1:

The damped harmonic oscillator (DHO) satisfies the following differential equation:

$$\frac{d^2x}{dt^2}+\frac{c}{m}\frac{dx}{dt}+\frac{k}{m}x = 0$$

It differs from the previous example by the addition of the $(c/m) dx/dt$ term. Like we did above, we can unwrap this second-order ODE into two first-order ODEs using two separate variables $x(t)$ and $v(t)$

\begin{align}
x' &= v \\
v' &= -\frac{c}{m}v - \frac{k}{m}x
\end{align}

1. Like in the example above, write down the update rules for $x_i$ and $v_i$.
1. Then write some code to implement your rules to estimate a numerical solution for $x(t)$ and $v(t)$ for a given initial condition $x_0$ and $v_0$ (you can assume $t_0 = 0$ like above).
1. Plot your results for $x(t)$ and $v(t)$ and make sure that they make sense. You may use the code in the example as a template.

*Hint*: Recall that the qualitative behavior of the oscillator is different depending on the (dimensionless) value of the ratio

$$\frac{(c/m)^2}{k/m}$$

So you should be able to see the effect of this by trying out different values for $c/m$ and $k/m$.

```python
# Code for Exercise 1
```

## But wait...

But you know that for a closed system, like the SHO, we actually have a special constraint on the system--the total energy (kinetic + potential) must be constant! So at every point of our solution, we should check whether this is true. How do we evaluate the total energy?

$$E = T + U = \frac{1}{2}mv^2 + \frac{1}{2}kx^2$$

Let's define a rescaled energy $\tilde{E}$ as $(1/m)E$:

$$\tilde{E} = \frac{1}{2}v^2 + \frac{1}{2}\frac{k}{m} x^2$$

### Problem 2:

1. Copy the code from the example using the SHO above, in which we solved the SHO using the Euler Method. Add code to calculate the rescaled energy $\tilde{E}_i$ for each time step.
1. Plot $\tilde{E}(t)$ vs. the time. Does the energy stay constant, fluctuate around some constant value, or does it diverge/decay?

```python
# Code for Exercise 2
```

## Euler-Cromer/Symplectic Euler Method

There exists a way to keep the energy fluctuations from growing, using a just a slight variant of the update rules described above. This update rule is called the

\begin{align}
v_i &= v_{i-1} + \left(-\frac{k}{m}x_{i-1}\right)\Delta t \\
x_i &= x_{i-1} + v_{i} \Delta t
\end{align}

In this version, you use the approximate velocity at time $t_i$ instead of the velocity at time $t_{i-1}$ to calculate $x_i$.

### Problem 3:

1. Modify the code from Exercise 2 to instead implement the update rule in the Euler-Cromer method. You can either modify the it in-place or copy it to the cell below and modify it.
1. Now run your code to calculate and plot $x(t)$, $v(t)$, and $\tilde{E}(t)$. Does the energy stay constant, fluctuate around some constant value, or does it diverge/decay?

```python
# Code for Exercise 3
```

There are also higher order ODE integration schemes, like Runge-Kutta, which make better estimates of the change in $(x(t),v(t)...)$ between $t_{i-1}$ and $t_i$. The shortcoming of our simple method above is that we are typically using the value of the derivative ($x'$ or $v'$) at $t_i$ or $t_{i-1}$ as a subsitute for the derivative over the entire interval $(t_{i-1}, t_i)$. These higher order schemes try to make better estimates of the derivatives inside this interval to make a better estimate of $\Delta x$ and $\Delta v$.

## Visualizing Phase Space

Here we generalize the use of the slope field above to visualize our error in the Euler method. The tool below is called a phase portrait and is ubiquitous in physics and mathematics, and students studying dynamical systems for their capstone projects may find it useful as a nice visualization. In the cell below, we examine the phase portrait of the SHO and study the numerical and analytical solutions. Before you run this cell, run the SHO example cell again with `x_0 = 1.0` and `v_0 = 0.0` so that `km`, `x_data`, and `v_data` are properly populated.

```python
import numpy as np
%matplotlib inline
import matplotlib.pyplot as plt

xvalues, yvalues = np.meshgrid(np.arange(min(x_data),max(x_data), 0.2), np.arange(min(v_data), max(v_data), 0.5))
xdot = yvalues
ydot = - km * xvalues

plt.figure(figsize=(8,8))
plt.streamplot(xvalues, yvalues, xdot, ydot)

plt.plot(x_data[0],v_data[0],'go',markersize=8)
plt.plot(x_data, v_data,color='r', label="numerical")
plt.plot(analytical_x, analytical_v, color='k', label="analytical")
plt.ylabel("Velocity")
plt.xlabel("Position")
plt.title("Phase Portrait")
plt.legend()
plt.grid()
plt.show()
```

You can make phase portraits for just about any system! Here's a phase portrait for the DHO. How does the phase portrait change qualitatively, as you vary $c/m$ and $k/m$?

```python
cm = 0.2 # c / m
km = 0.3 # k / m

xvalues, yvalues = np.meshgrid(np.arange(-3,3, 0.5), np.arange(-3,3, 0.5))
xdot = yvalues
ydot = - cm * yvalues - km * xvalues

plt.figure(figsize=(8,8))
plt.streamplot(xvalues, yvalues, xdot, ydot)

plt.ylabel("Velocity")
plt.xlabel("Position")
plt.title("Phase Portrait")
plt.grid()
plt.show()
```

```python

```


Massimo Nocentini

March 2, 2018: compositional inverse

February 26, 2018: splitting from generic nb

Abstract

Ctors for (symbolic) Riordan arrays.

```python
from sympy import *
from sympy.abc import n, i, N, x, lamda, phi, z, j, r, k, a, t, alpha

from sequences import *

init_printing()
```

```python
m = 5
d_fn, h_fn = Function('d'), Function('h')
d, h = IndexedBase('d'), IndexedBase('h')
```

---

```python
rows, cols = 5, 5
ctor = lambda i,j: d[i,j]
Matrix(rows, cols, ctor)
```

## By series convolution

```python
d_series = Eq(d_fn(t), 1+sum(d[i]*t**i for i in range(1,m)))
h_series = Eq(h_fn(t), t*(1+sum(h[i]*t**i for i in range(1,m-1)))).expand()

d_series, h_series
```

```python
R = Matrix(m, m, riordan_matrix_by_convolution(m, d_series, h_series))
R
```

```python
production_matrix(R) # too verbose to show
```

```python
d_series = Eq(d_fn(t), 1/(1-t))
h_series = Eq(h_fn(t), t*d_series.rhs)

d_series, h_series
```

```python
R = Matrix(10, 10, riordan_matrix_by_convolution(10, d_series, h_series))
R
```

## By recurrence relation

```python
dim = 5
a, b, b_bar, c = symbols(r'a b \bar{b} c')
M = Matrix(dim, dim,
           riordan_matrix_by_recurrence(
               dim, lambda n, k: {(n-1, k-1):a,
                                  (n-1, k): b if k else b_bar,
                                  (n-1, k+1):c}))
M
```

```python
production_matrix(M)
```

```python
Msubs = M.subs({a:1, b_bar:b})
Msubs, production_matrix(Msubs)
```

## By $A, Z$ sequences

### $\mathcal{P}$

```python
A, Z = Function('A'), Function('Z')
A_eq = Eq(A(t), 1 + t)
Z_eq = Eq(Z(t),1)
A_eq, Z_eq
```

```python
R = Matrix(10, 10, riordan_matrix_by_AZ_sequences(10, (Z_eq, A_eq)))
R, production_matrix(R)
```

### $\mathcal{C}$

```python
A = Function('A')
A_ones = Eq(A(t), 1/(1-t))
R = Matrix(10, 10, riordan_matrix_by_AZ_sequences(10, (A_ones, A_ones)))
R, production_matrix(R)
```

### $\mathcal{R}$

```python
dim = 5
A = Function('A')
a = IndexedBase('a')
A_gen = Eq(A(t), sum((a[j] if j else 1)*t**j for j in range(dim)))
R = Matrix(dim, dim, riordan_matrix_by_AZ_sequences(dim, (A_gen, A_gen)))
R
```

```python
z = IndexedBase('z')
A_gen = Eq(A(t), sum((a[j] if j else 1)*t**j for j in range(dim)))
Z_gen = Eq(Z(t), sum((z[j] if j else 1)*t**j for j in range(dim)))
Raz = Matrix(dim, dim, riordan_matrix_by_AZ_sequences(dim, (Z_gen, A_gen)))
Raz
```

```python
production_matrix(R), production_matrix(Raz)
```

## Compositional inverse

```python
H = Function('h')
C_eq = Eq(H(t), (1-sqrt(1-4*t))/2)
C_eq, compositional_inverse(C_eq)
```

```python
P_eq = Eq(H(t), t/(1-t))
(P_eq,
 compositional_inverse(P_eq),
 compositional_inverse(compositional_inverse(P_eq), y=t))
```

## Group inverse

```python
d_series = Eq(d_fn(t), 1/(1-t))
h_series = Eq(h_fn(t), t/(1-t))
P_inverse = group_inverse(d_series, h_series)
P_inverse
```

```python
R = Matrix(10, 10, riordan_matrix_by_convolution(10, *P_inverse))
R, production_matrix(R)
```

```python
catalan_term = (1-sqrt(1-4*t))/(2*t)
d_series = Eq(d_fn(t), catalan_term)
h_series = Eq(h_fn(t), t*catalan_term)
C_inverse = group_inverse(d_series, h_series, post=radsimp)
C_inverse
```

```python
R = Matrix(10, 10, riordan_matrix_by_convolution(10, C_inverse[0], C_inverse[1]))
R
```

## Exponential RA

build the triangle of Stirling numbers of the II kind

```python
d_series = Eq(d_fn(t), 1)
h_series = Eq(h_fn(t), exp(t)-1)
d_series, h_series
```

```python
R = Matrix(10, 10, riordan_matrix_exponential(
            riordan_matrix_by_convolution(10, d_series, h_series)))
R
```

```python
production_matrix(R), production_matrix(R, exp=True)
```

```python
inspect(R)
```

    nature(is_ordinary=False, is_exponential=True)

---
https://oeis.org/A021009

```python
d_series = Eq(d_fn(t), 1/(1-t))
h_series = Eq(h_fn(t), t/(1-t))
d_series, h_series
```

```python
R = Matrix(10, 10, riordan_matrix_exponential(
            riordan_matrix_by_convolution(10, d_series, h_series)))
R
```

```python
production_matrix(R), production_matrix(R, exp=True)
```

```python
inspect(R)
```

    nature(is_ordinary=False, is_exponential=True)

---

This work is licensed under a Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License.


```python
import numpy as np
import matplotlib.pyplot as plt
import sympy
from sympy import *
init_printing(use_unicode=True)
```

## 5.28

```python
def wave_packet(a,b):
    x,t=symbols('x t')
    express=exp(-(x - 3 * t) ** 2)*sin(3 * pi * (x - t))
    ret=express.subs({x:a,t:b})

    return ret.evalf()

xlist = np.linspace(-4, 4, 1000)
ylist=[]
for i in xlist:
    ylist.append(wave_packet(i, 0))

plt.plot(xlist, ylist)
plt.xlabel('x')
plt.ylabel('Amplitude')
plt.show()
```

## 5.29

```python
import numpy as np
x = np.linspace(0, 2, 20)
y = x*(2 - x)
import matplotlib.pyplot as plt
plt.plot(x, y)
plt.show()
```

### After switching to y=cos(18 * pi * x) and using sympy

```python
def func(x_val):
    x=Symbol('x')
    expr=cos(18 * pi * x)
    ret=expr.subs(x,x_val)
    return ret.evalf()

x = np.linspace(0,2,20)
X = np.linspace(0,2,1000)
y=[]
Y=[]
for i in x:
    y.append(func(i))
for i in X:
    Y.append(func(i))
plt.plot(x,y)
plt.plot(X,Y)
plt.title('Plotting f(x)=cos(18*pi*x)')
plt.legend(['20 points', '2000 points'])
plt.xlabel('x')
plt.ylabel('f(x)')
plt.show()
```

#### The blue curve is less accurate and the orange one is more accurate as it has more points (desmos plot example given above)

## 5.30

```python
def viscosity(t_val):
    t=Symbol('t')
    exprr= 2.414e-5 * 10 ** (247.8 / (t + 273.15 - 140.0))
    ret=exprr.subs(t,t_val)
    return ret.evalf()

T = np.linspace(0, 100, 1000)
visc=[]
for i in T:
    visc.append(viscosity(i))

plt.plot(T, visc)
plt.ylabel('viscosity (Pa s)')
plt.xlabel('temperature (C)')
plt.show()
```

## 5.31

```python
def c(lamb):
    l=Symbol('l')
    expp=sqrt(9.81 * l / (2 * pi) *(1 + 7.9e-2 * 4 * pi ** 2 / (1000 * 9.81 * l ** 2)) *tanh(2 * pi * 50.00 / l))
    ret=expp.subs(l,lamb)
    return ret.evalf()

l = np.linspace(0.001, 0.1, 1000)
c_arr1=[]
for i in l:
    c_arr1.append(c(i))
plt.plot(l, c_arr1)
plt.xlabel('wavelength (m)')
plt.ylabel('speed (m/s)')
plt.show()

l = np.linspace(1, 2000, 1000)
c_arr2=[]
for i in l:
    c_arr2.append(c(i))
plt.plot(l, c_arr2)
plt.xlabel('wavelength (m)')
plt.ylabel('speed (m/s)')
plt.show()
```

```python

```


```python
import matplotlib.pyplot as plt
import numpy as np
from mpl_toolkits.mplot3d import Axes3D
import scipy as sp
import scipy.linalg
import sympy as sy
sy.init_printing()
```

#  Vector Space

A vector space, as its name indicates, is the space for vectors, which defines two operations, **addition** and **multiplication by scalars**, subject to $10$ axioms below.

1. The sum of ${u}$ and ${v}$, denoted by ${u}+{v},$ is in $V$
2. ${u}+{v}={v}+{u}$
3. $({u}+{v})+{w}={u}+({v}+{w})$
4. There is a zero vector $0$ in $V$ such that ${u}+{0}={u}$
5. For each ${u}$ in $V$, there is a vector $-{u}$ in $V$ such that ${u}+(-{u})={0}$
6. The scalar multiple of ${u}$ by $c,$ denoted by $c {u},$ is in $V$
7. $c({u}+{v})=c {u}+c {v}$
8. $(c+d) {u}=c {u}+d {u}$
9. $c(d {u})=(c d) {u}$
10. $1 {u}={u}$

Though $10$ axioms seem quite apparent and superfluous, simply remember this: _addition and multiplication are closed in vector space_.

All axioms are self-explanatory without proof, we can visualize axiom $7$ with a user-defined function.

```python
def vecSpaceAx7(u, v, c):
    fig, ax = plt.subplots(figsize = (7, 7))
    '''Syntax vecSpaceAx7(u, v, c), to demonstrate Axiom 7.'''
    ax.arrow(0, 0, u[0], u[1], color = 'red', width = .08,
         length_includes_head = True,
         head_width = .3, # default: 3*width
         head_length = .6,
         overhang = .4)

    ax.arrow(0, 0, v[0], v[1], color = 'blue', width = .08,
         length_includes_head = True,
         head_width = .3, # default: 3*width
         head_length = .6,
         overhang = .4)

    ax.arrow(0, 0, u[0]+v[0], u[1]+v[1], color = 'green', width = .08,
         length_includes_head = True,
         head_width = .3, # default: 3*width
         head_length = .6,
         overhang = .4)

    ax.arrow(0, 0, c*u[0], c*u[1], color = 'red', width = .08, alpha=.5,
         length_includes_head = True,
         head_width = .3, # default: 3*width
         head_length = .6,
         overhang = .4)

    ax.arrow(0, 0, c*v[0], c*v[1], color = 'blue', width = .08, alpha=.5,
         length_includes_head = True,
         head_width = .3, # default: 3*width
         head_length = .6,
         overhang = .4)
    ax.arrow(0, 0, c*(u[0]+v[0]), c*(u[1]+v[1]), color = 'green', width = .08, alpha=.5,
         length_includes_head = True,
         head_width = .3, # default: 3*width
         head_length = .6,
         overhang = .4)

    ###########################Dashed Lines#################################
    point1 = [u[0], u[1]]
    point2 = [u[0] + v[0], u[1] + v[1]]
    line = np.array([point1, point2])
    ax.plot(line[:,0], line[:,1], ls = '--', lw = 3, color = 'red', alpha = .5)

    point1 = [v[0], v[1]]
    point2 = [u[0] + v[0], u[1] + v[1]]
    line = np.array([point1, point2])
    ax.plot(line[:,0], line[:,1], ls = '--', lw = 3, color = 'blue', alpha = .5)

    point1 = [c*v[0], c*v[1]]
    point2 = [c*(u[0] + v[0]),c*(u[1] + v[1])]
    line = np.array([point1, point2])
    ax.plot(line[:,0], line[:,1], ls = '--', lw = 3, color = 'blue', alpha = .5)

    point1 = [c*u[0], c*u[1]]
    point2 = [c*(u[0] + v[0]), c*(u[1] + v[1])]
    line = np.array([point1, point2])
    ax.plot(line[:,0], line[:,1], ls = '--', lw = 3, color = 'blue', alpha = .5)

    ####################################Text###############################

    ax.text(x = u[0], y = u[1], s = '$(%.0d, %.0d)$' % (u[0], u[1]), size = 16)
    ax.text(x = v[0], y = v[1], s = '$(%.0d, %.0d)$' % (v[0], v[1]), size = 16)
    ax.text(x = u[0]+v[0], y = u[1]+v[1], s = '$(%.0d, %.0d)$' % (u[0]+v[0], u[1]+v[1]), size = 16)
    ax.text(x = c*u[0], y = c*v[1], s = '$(%.0d, %.0d)$' % (c*u[0], c*u[1]), size = 16)
    ax.text(x = c*v[0], y = c*v[1], s = '$(%.0d, %.0d)$' % (c*v[0], c*v[1]), size = 16)
    ax.text(x = c*(u[0]+v[0]), y = c*(u[1]+v[1]), s = '$(%.0d, %.0d)$' % (c*(u[0]+v[0]), c*(u[1]+v[1])), size = 16)

    ax.set_title('Axiom 7: $c(\mathbf{u}+\mathbf{v})=c \mathbf{u}+c \mathbf{v}$', size = 19, color = 'red')

    ax.axis([0, np.max(c*u)+6, 0, np.max(c*v)+6])
    ax.grid(True)

if __name__ == '__main__':
    u = np.array([2,3])
    v = np.array([3,1])
    c = 2
    vecSpaceAx7(u, v, c)
```

Try different vectors.

```python
u = np.array([1, 3])
v = np.array([5, 2])
vecSpaceAx7(u, v, 2)
```

However the vector space has more general meaning than containing vectors, the functions and polynomials can also be in the vector space as we have show in the section of linear dependence.

The difference is that the functions has infinite number of elements (continuous functions) in contrast to vectors.

We can demonstrate in the vector space for functions by plotting two trigonometric functions: $\sin(x)$ and $\cos{(x+1)}$ (stem plot), their addition in vector space is the pink shaded plot.

```python
def funcVecSp(x1, x2, c):
    fig, ax = plt.subplots(figsize = (10, 10))
    '''Syntax funcVecSp(x), x is the domain, for instance, x = np.linspace(-3, 3, 20)'''
    y1 = c * np.sin(x1)
    ax.stem(x1, y1, use_line_collection = True, linefmt = 'red', label ='$\sin{x}$')

    y2 = c * np.cos(x2+1)
    ax.stem(x2, y2, use_line_collection = True, linefmt = 'blue')

    y3 = c*(np.sin(x1)+np.cos(x2+1))
    ax.plot(x2, y3, lw = 4, color = 'red', alpha = .6)

    ax.fill_between(x2, y3, 0, color  = 'red', alpha = .3)

    str1 = '$\sin(x)+\cos(x+1)$'
    ax.annotate(str1, xy = (1, 2*(np.sin(1)+np.cos(1+1))), xytext = (0, 3), weight = 'bold', color  =  'r', fontsize =18,
             arrowprops = dict(arrowstyle = '->', connectionstyle = 'arc3', color = 'b'))

    ax.axis([-5, 5, -4, 4])
    ax.legend()

if __name__ == '__main__':
    x1 = np.arange(-5, 5, 0.3)
    x2 = np.arange(-5.1, 4.9, 0.3)
    c = 2
    funcVecSp(x1, x2, c)
```

So we can say that $\sin{(x)}+\cos{(x+1)}$ is in the same vector space of $\sin{(x)}$ and $\cos{(x+1)}$.

#  Subspace

A subspace is one of the most important concept in linear algebra, fortunately nothing really mysterious.

A subspace resides in a vector space $V$,  we can denoted it as $H$. Only two properties needs to be verified:
1. $H$ has zero vector.
2. Closed under vector addition and scalar multiplication.

There are two facts of subspace:
1. Any line passes through $(0, 0)$ in $\mathbb{R}^2$ is a subspace of $\mathbb{R}^2$.
2. Any plane passes through $(0, 0, 0)$ in $\mathbb{R}^3$ is a subspace of $\mathbb{R}^3$.

Next we will visualize the subspace.

##  Visualization of Subspace of $\mathbb{R}^2$

```python
fig, ax = plt.subplots(figsize = (10, 10))
####################### Arrows #######################
x = np.arange(-10, 11, 1)
y = 4/6*x
ax.plot(x, y, lw = 4, color = 'blue',label = r'$y = \frac{2}{3}x$, subspace of $\mathbf{R}^2$')

y = -3+4/6*x
ax.plot(x, y, lw = 4, color = 'red',label = r'$y = -3+\frac{2}{3}x$, not a subspace of $\mathbf{R}^2$')

ax.grid(True)
ax.set_title('Visualization of Subspace in $R^2$ ', size = 18)
ax.scatter(0, 0, s= 100, fc = 'black', ec = 'red')
ax.text(-2, 0, '$(0,0)$',size = 18)
ax.legend(fontsize = 16)

ax.axis([-10, 10, -10, 10])

ax.set_xlabel('x-axis', size = 18)
ax.set_ylabel('y-axis', size = 18)
plt.show()
```

##  Visualization of Subspace of $\mathbb{R}^3$

Consider a span of two vectors $u = (1,-2,1)^T$ and $v=(2,1,2)^T$. The span of $(u,v)$ is a subspace of $\mathbb{R}^3$, where $s$ and $t$ are the scalars of the vectors.

$$
\left[
\begin{matrix}
x\\
y\\
z
\end{matrix}
\right]=
s\left[
\begin{matrix}
1\\
-2\\
1
\end{matrix}
\right]+
t\left[
\begin{matrix}
2\\
1\\
2
\end{matrix}
\right]=
\left[
\begin{matrix}
s+2t\\
-2s+t\\
s+2t
\end{matrix}
\right]
$$

We also plot a plan which is not a subspace by adding $2$ onto the third equation, i.e. $z= s+2t+2$.

Remember matplotlib does not have 3D engine, we have to pan to a proper angle to show the layout.

```python
#%matplotlib notebook, use this only if you are in Jupyter Notebook
fig = plt.figure(figsize = (8,8))

ax = fig.add_subplot(111,projection='3d')

s = np.linspace(-1, 1, 10)
t = np.linspace(-1, 1, 10)
S, T = np.meshgrid(s, t)

X = S+2*T
Y = -2*S+T
Z = S+2*T

ax.plot_surface(X, Y, Z, alpha = .9,cmap=plt.cm.coolwarm)

Z2 = S+2*T+2 # this is not a subspace anymore
ax.plot_surface(X, Y, Z2, alpha = .6 ,cmap=plt.cm.jet)

ax.scatter(0,0,0, s = 200, color = 'red')
ax.text(0,0,0,'$(0,0,0)$',size = 18, zorder = 5)

ax.set_title('Visualization of Subspace of $\mathbb{R}^3$', x = .5, y = 1.1, size = 20)

ax.set_xlabel('x-axis', size = 18)
ax.set_ylabel('y-axis', size = 18)
ax.set_zlabel('z-axis', size = 18)

ax.view_init(elev=-29, azim=132)
plt.show()
```

As you can see the plane contains $(0,0,0)$ is a subspace, but the other plane is not.

#  Span

We have mentioned **span** quite a few times before, now we give formal definition of a span.

Use $\text{span}(S)$ to denote span of a subset $\{v_1, v_2,...,v_n\}$, which is a linear combination in vector space $V$.

$$
\text{span}(S)=\{c_1v_1+c_2v_2...+c_nv_n:c_1,...,c_n \in R\}
$$

The span of two vectors in $\mathbb{R}^3$ is a plane and any two vectors span a plane. Say, we have two vectors: $(3, 9, 2)$, $(1,7,5)$, any linear combination is a span, i.e. $s(3,9,2)^T+t(1,7,5)^T$.

For more general span, a basic fact of matrix multiplication can assist us in demonstrating:

$$
AB = A[b_1\ b_2\  b_3,...,b_4]=[Ab_1\ Ab_2\ Ab_3,...,Ab_p]
$$

where $A$ is the spanning set of vectors, $b_k$ is vector of weights for linear combination.We can generate a random matrix $B$ to form various linear combinations to visually verify if they are all contained in the spanned plane.

We define
$$
A=\left[\begin{array}{rr}
3 & 1 \\
9 & 7 \\
2 & 5
\end{array}\right]\qquad b_i\sim N(\mathbb{0}, 1)
$$

```python
A = np.array([[3,9,2],[1,7,5]]).T

B = 10*np.random.randn(2, 300) # i = 300, i.e. 300 random weight vectors
vecs = A@B

s = np.linspace(-10, 10, 10)
t = np.linspace(-10, 10, 10)

S, T = np.meshgrid(s, t)

X = 3*S+T
Y = 9*S+7*T
Z = 2*S+5*T

fig, ax = plt.subplots(figsize = (10, 10))
ax = fig.add_subplot(projection='3d')
ax.plot_wireframe(X, Y, Z, linewidth = 1.5, color = 'k', alpha = .6)
ax.scatter(0,0,0, s =200, ec = 'red', fc = 'black')

colors = np.random.rand(vecs.shape[1],3)
for i in range(vecs.shape[1]):
    vec = np.array([[0, 0, 0, vecs[0,i], vecs[1,i], vecs[2,i]]])
    X, Y, Z, U, V, W = zip(*vec)
    ax.quiver(X, Y, Z, U, V, W, length=1,color = colors[i], normalize=False, arrow_length_ratio = .07, pivot = 'tail',
          linestyles = 'solid',linewidths = 2, alpha = .9)
ax.view_init(elev=-156, azim=-56)
plt.show()
```

Pan around the plot, we confirm that all the vectors are in the $\text{Span}(u,v)$.

##  Span of $\mathbb{R}^3$

Reproduce the code above, but we have three vectors: $(1,0,1)$, $(1,1,0)$, $(0,1,1)$. Again we create a random coefficent matrix to form different linear combinations.

```python
A = np.array([[1,0,1],[1,1,0],[0,1,1]]).T
B = 5*np.random.randn(3, 300)
vecs = A@B

s = np.linspace(-10, 10, 10)
t = np.linspace(-10, 10, 10)

S, T = np.meshgrid(s, t)

X = S+T
Y = T
Z = S

fig, ax = plt.subplots(figsize = (10, 10))
ax = fig.add_subplot(projection='3d')
ax.plot_wireframe(X, Y, Z, linewidth = 1.5, color = 'k', alpha = .6)
ax.scatter(0,0,0, s =200, ec = 'red', fc = 'black')

colors = np.random.rand(vecs.shape[1],3)
for i in range(vecs.shape[1]):
    vec = np.array([[0, 0, 0, vecs[0,i], vecs[1,i], vecs[2,i]]])
    X, Y, Z, U, V, W = zip(*vec)
    ax.quiver(X, Y, Z, U, V, W, length=1,color = colors[i], normalize=False, arrow_length_ratio = .07, pivot = 'tail',
          linestyles = 'solid',linewidths = 2, alpha = .9)
ax.view_init(elev=21, azim=-110)
plt.show()
```

The vectors are pointing every possible directions in $\mathbb{R}^3$, and all of them stay in the span of those three vectors.

Chapter 8 - Vector Space and Subspace.ipynb


Ex. 1: Simulação de uma rede simples de distribuição
---------------

```python
import numpy as np
from sympy import nsolve, sqrt, log, pi, symbols, Array, log
```

```python
# Bellos, Nalbantis, Tsakiris (2018)
def f(Re, eD):
    """"""
    a = 1/(1 + pow(Re/2712, 8.4))
    b = 1/(1 + pow(Re/(150/eD), 1.8))
    A = pow(64/Re, a)
    B = pow(0.75*log(Re/5.37), 2*(a-1)*b)
    C = pow(0.88*log(6.82/eD), 2*(a-1)*(1-b))
    return A*B*C

def Re(Q, D, nu=1e-6):
    ""
    return 4*Q/(nu*pi*D)
```

```python
D = 0.15
G = 1e4
g = 9.81
h = 1.0
H = 6.0
eD = 0.26e-3/D

La, Lb, Lc, Ld, Le = symbols('La Lb Lc Ld Le')
Qa, Qb, Qc, Qd, Qe = symbols('Qa Qb Qc Qd Qe')
p0, p1, p2, p3, p4, p5, p6 = symbols('p0 p1 p2 p3 p4 p5 p6')
p1 = H*G
p3 = p5 = p6 = 0

Qa = 0.045
Qb = 0.021
Qc = Qa - Qb
Qd = 0.017
Qe = Qc - Qd

La = 6
Lc = 4

Eqa = p1/G - p2/G - f(Re(Qa,D),eD)*8*Qa*Qa*La/(pi*pow(D,5)*g)
Eqb = p2/G - p3/G - f(Re(Qb,D),eD)*8*Qb*Qb*Lb/(pi*pow(D,5)*g)
Eqc = p2/G - p4/G - f(Re(Qc,D),eD)*8*Qc*Qc*Lc/(pi*pow(D,5)*g) + h
Eqd = p4/G - p6/G - f(Re(Qd,D),eD)*8*Qd*Qd*Ld/(pi*pow(D,5)*g)
Eqe = p4/G - p5/G - f(Re(Qe,D),eD)*8*Qe*Qe*Le/(pi*pow(D,5)*g)

ans = nsolve([Eqa, Eqb, Eqc, Eqd, Eqe],
             [p4, p2, Lb, Ld, Le],
             [1, 1e-1, 1e-1, 1, 1.2], verify=False)
p4, p2, Lb, Ld, Le = list(ans)
list(ans)
```

    [60870.9940289691,
     52312.9245743828,
     189.681851495541,
     336.172800914279,
     1823.47218863677]

### Desenho da linha
-------------------

```python
%matplotlib notebook
from pylab import *
from mpl_toolkits.mplot3d import Axes3D
from matplotlib.collections import LineCollection
fig = figure(figsize=(8,8))
ax = fig.add_subplot(111, projection='3d')

lc = sqrt(Lc*Lc-h*h)
X = [0, La, La, La+lc, La+lc, La+Ld+lc]
Y = [0, 0,  Lb, 0, -Le, 0]
Z = [h, h,  h,  0, 0, 0]

ax.plot(X[:2], Y[:2], Z[:2])
ax.plot(X[1:3], Y[1:3], Z[1:3])
ax.plot(X[1:4:2], Y[1:4:2], Z[1:4:2])
ax.plot(X[3:5], Y[3:5], Z[3:5])
ax.plot(X[3:6:2], Y[3:6:2], Z[3:6:2])

ax.set_xlabel('X')
ax.set_ylabel('Y')
ax.set_zlabel('Z')
ax.set_zlim([-10,10])
```

    <IPython.core.display.Javascript object>

    (-10, 10)


$\newcommand{\xv}{\mathbf{x}}
 \newcommand{\wv}{\mathbf{w}}
 \newcommand{\yv}{\mathbf{y}}
 \newcommand{\zv}{\mathbf{z}}
 \newcommand{\Chi}{\mathcal{X}}
 \newcommand{\R}{\rm I\!R}
 \newcommand{\sign}{\text{sign}}
 \newcommand{\Tm}{\mathbf{T}}
 \newcommand{\Xm}{\mathbf{X}}
 \newcommand{\Xlm}{\mathbf{X1}}
 \newcommand{\Wm}{\mathbf{W}}
 \newcommand{\Vm}{\mathbf{V}}
 \newcommand{\Ym}{\mathbf{Y}}
 \newcommand{\Zm}{\mathbf{Z}}
 \newcommand{\Zlm}{\mathbf{Z1}}
 \newcommand{\I}{\mathbf{I}}
 \newcommand{\muv}{\boldsymbol\mu}
 \newcommand{\Sigmav}{\boldsymbol\Sigma}
 \newcommand{\Phiv}{\boldsymbol\Phi}
$

# Neural Networks

Neural networks, or artificial neural networks, are the computational models inspired by the brain. Mimicing the neurons' synaptic connecions (Figure 1), we build or stack multiple neuron-like hidden units to map data into nonlinear space for rich representation.

<center>Figure 1. Anatomy of a neuron (wikipedia) </center>

Now, let us review the perceptron model.

In perceptron, passing the output of linear model to the step function, we get discrete outputs.
Now, you can think a perceptron as a neuron. With a threshold zero, when the linear model outputs are over it, it passes the signal to next neuron.

By connecting the perceptrons, we can actually build synaptic connections.
We call this model as *multi-layer perceptron* (MLP).

**Q:** For inputs $x \in \{-1, +1 \}$, think about what the following picture represents and answer for it.

1)

2)

3)

Answer:
1)

2)

3)

## Feed Forward Neural Networks

Fitting the data with MLP is a combinatorial optimization problem with non-smooth step function.
So, we can consider smooth step function, a s-shaped sigmoid function.
We call this smooth function as **activation function**.

```python
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline

fig, ax = plt.subplots()

# x - y axis
ax.axhline(y=0, color='k', linewidth=1)
ax.axvline(x=0, color='k', linewidth=1)

# step function in blue
plt.plot([0, 6], [1, 1], 'b-', linewidth=3)
plt.plot([-6, 0], [-1, -1], 'b-', linewidth=3)
plt.plot([0, 0], [-1, 1], 'b-', linewidth=3)

# tanh in red
x = np.linspace(-6, 6, 100)
plt.plot(x, np.tanh(x), 'r-', linewidth=3)

```

## Non-linear Extension of Linear Model

As we discussed, feed forward neural networks have a rich representation. Thus, it can represent the linear model with single layer.

Considering the multiple outputs, we formulated this in matrix:

$$
\begin{align}
E &= \frac{1}{N} \frac{1}{K} \sum_{n=1}^{N} \sum_{k=1}^{K} (t_{nk} - y_{nk})^2 \\
\\
\Ym &= \Xlm \cdot \Wm
\end{align}
$$

Here, we assume the first column of $\Xlm$ is the bias column with 1's.
Thus, the weight matrix $\Wm$ is $(D+1) \times K$ with the bias row in the first row.

From this model, we can convert the raw data $\Xm$ to $\Phiv$, which is a nonlinear mapping.

$$
\phi: \Xm \rightarrow \Phiv
$$

Then, we can rewrite the linear model with as follows:

$$
\begin{align}
E &= \frac{1}{N} \frac{1}{K} \sum_{n=1}^{N} \sum_{k=1}^{K} (t_{nk} - y_{nk})^2
\\
\Ym &= \Phiv \Wm \\
\\
\Ym_{nk} &= \Phiv_n^\top \Wm_k
\end{align}
$$

Now, let $\phi(\xv) = h(\xv)$ where $h$ is the *activation function*.

$$
\begin{align}
\Zm &= h(\Xlm \cdot \Vm) \\
\\
\Ym & = \Zlm \cdot \Wm
\end{align}
$$

Figure below depics this model.

The size of each matrix is listed:
- $\Xm: N \times D$
- $\Xlm: N \times (D+1)$
- $\Vm: (D+1) \times G$
- $\Zm: N \times G$
- $\Zlm: N \times (G+1)$
- $\Wm: (G+1) \times K$
- $\Ym: N \times K$

For this two-layer network, we call the blue circle layer with the activation functions as **hidden layer** and the organge layer with summation as **output layer**.

# Why Sigmoid?

The resemblance to the step function can be good reason. But is there any other reason for choosing a sigmoid function as activation?

Let us take a look at a polinomial function and the sigmoid.

$$
y = x^4 + 3 x^2 + 7 x + 3   \quad\quad\text{vs.}\quad\quad y = tanh(x)
$$

```python

# polinomial function
def h_poly(x):
    return x**4 + 3 * x**2 + 7 * x + 3

# polinomial function
def h_sigmoid(x):
    return np.tanh(x)

##### Gradient functions

# polinomial function
def dh_poly(x):
    return 4 * x**3 + 6 * x + 7

# polinomial function
def dh_sigmoid(x):
    h = h_sigmoid(x)
    return 1 - h ** 2

x = np.linspace(-6, 6, 100)

plt.figure(figsize=(16,8))
plt.subplot(121)
plt.plot(x, h_poly(x), label="$y = x^4 + 3 x^2 + 7 x + 3$")
plt.plot(x, dh_poly(x), label="$dy$")
plt.legend()

plt.subplot(122)
plt.plot(x, h_sigmoid(x), label="$y = tanh(x)$")
plt.plot(x, dh_sigmoid(x), label="$dy$")
plt.legend()

```

Here, we can see the polinomial gradients are very huge when $x$ is moving away from 0. A gradient descent producure takes this huge step for the large positive or negative $x$ values, which can make learning divergent and unstable.

In the right figure, we can see the gradient is nearly turned off for large $x$ values. Only on the nonlinear region of sigmoid function, small gradient is applied for stable learning.

# Gradient Descent

From the error function $E$,

$$
E = \frac{1}{N} \frac{1}{K}\sum_{n=1}^{N} \sum_{k=1}^{K} (t_{nk} - y_{nk})^2,
$$

we can derive the gradient to update the weights for each layer.

Since we can change the output and eventually the error by changing the weights $\Vm$ and $\Wm$,

$$
\begin{align}
v_{dg} &\leftarrow v_{dg} - \alpha_h \frac{\partial{E}} {\partial{v_{dg}}} \\
\\
w_{gk} &\leftarrow w_{gk} - \alpha_o \frac{\partial{E}} {\partial{w_{gk}}},
\end{align}
$$

where $\alpha_h$ and $\alpha_o$ are the learning rate for hidden and output layer respectively.

$$
\begin{align}
\frac{\partial{E}}{\partial{w_{gk}}} &= \frac{\partial{\Big( \frac{1}{N} \frac{1}{K} \sum_{n=1}^{N} \sum_{l=1}^{K} (t_{nl} - y_{nl})^2} \Big)}{\partial{w_{gk}}} \\
   &= -2  \frac{1}{N} \frac{1}{K} \sum_{n=1}^{N}(t_{nk} - y_{nk}) \frac{\partial{y_{nl}}}{\partial{w_{gk}}}
\end{align}
$$

where

$$
y_{nl} = z1_{n}^\top w_{*l} = \sum_{g=0}^{G} z1_{ng} w_{gl} .
$$

The gradient for the ouutput layer can be computed as follows:
$$
\begin{align}
\frac{\partial{E}}{\partial{w_{gk}}} &= -2 \frac{1}{N} \frac{1}{K} \sum_{n=1}^{N} (t_{nk} - y_{nk})  z1_{nk} \\
    &= -2 \frac{1}{N} \frac{1}{K} \Zlm^\top (\Tm - \Ym).
\end{align}
$$

For the hidden layer,

$$
\begin{align}
\frac{\partial{E}}{\partial{v_{dg}}} &= \frac{\partial{\Big( \frac{1}{N} \frac{1}{K} \sum_{n=1}^{N} \sum_{l=1}^{K} (t_{nl} - y_{nl})^2} \Big)}{\partial{v_{dg}}} \\
   &= -2  \frac{1}{N} \frac{1}{K} \sum_{n=1}^{N} \sum_{l=1}^{K} (t_{nl} - y_{nl}) \frac{\partial{y_{nl}}}{\partial{v_{dg}}}
\end{align}
$$

where

$$
y_{nl} = \sum_{g=0}^{G} z1_{ng} w_{gl} = \sum_{g=0}^G w_{gl} h (\sum_{d=0}^D v_{dg} x1_{nd}) .
$$

Let $a_{ng} = \sum_{d=0}^D x1_{nd} v_{dg}$. Then, we can use a chain rule for the derivation.

$$
\begin{align}
\frac{\partial{E}}{\partial{v_{dg}}} &= -2  \frac{1}{N} \frac{1}{K} \sum_{n=1}^{N} \sum_{l=1}^{K} (t_{nl} - y_{nl}) \frac{\partial{\Big(  \sum_{q=0}^G w_{ql} h (\sum_{p=0}^D v_{pq} x1_{np}) \Big)}}{\partial{v_{dg}}} \\
    &= -2  \frac{1}{N} \frac{1}{K} \sum_{n=1}^{N} \sum_{l=1}^{K} (t_{nl} - y_{nl}) \sum_{q=0}^G w_{ql} \frac{\partial{\Big( h (\sum_{p=0}^D v_{pq} x1_{np}) \Big)}}{\partial{v_{dg}}} \\
    &= -2  \frac{1}{N} \frac{1}{K} \sum_{n=1}^{N} \sum_{l=1}^{K} (t_{nl} - y_{nl}) \sum_{q=0}^G w_{ql} \frac{\partial{h(a_{ng})}}{\partial{a_{ng}}} \frac{\partial{a_{ng}}}{\partial{v_{dg}}} \\
    &= -2  \frac{1}{N} \frac{1}{K} \sum_{n=1}^{N} \sum_{l=1}^{K} (t_{nl} - y_{nl}) \sum_{q=0}^G w_{ql} \frac{\partial{h(a_{ng})}}{\partial{a_{ng}}} x1_{nd}.
\end{align}
$$

When $h = tanh$,

$$
\frac{\partial{h(a_{ng})}}{\partial{a_{ng}}} = \frac{z_{ng}}{\partial{a_{ng}}} = (1 - z_{ng}^2).
$$

Thus,

$$
\frac{\partial{E}}{\partial{v_{dg}}} = -2  \frac{1}{N} \frac{1}{K} \sum_{n=1}^{N} \sum_{l=1}^{K} (t_{nk} - y_{nl}) \sum_{g=0}^G w_{gl} (1 - z_{ng}^2) x1_{nd}.
$$

Rewriting this in matrix form,

$$
\frac{\partial{E}}{\partial{v_{dg}}} = -2 \frac{1}{N} \frac{1}{K} \Xlm^\top \Big( (\Tm - \Ym) \Wm^\top \odot (1 - \Zm^2) \Big).
$$

Here, $\odot$ denotes the element-wise multiplication.

To summarize, the backpropagation performs the this weight updates iteratively:
$$
\begin{align}
\Vm &\leftarrow \Vm + \rho_h \frac{1}{N} \frac{1}{K} \Xlm^\top \Big( (\Tm - \Ym) \Wm^\top \odot (1 - \Zm^2) \Big), \\
\Wm &\leftarrow \Wm + \rho_o \frac{1}{N} \frac{1}{K} \Zlm^\top \Big( \Tm - \Ym \Big)
\end{align}
$$
where $\rho_h$ and $\rho_o$ are the learning rate for hidden and output layer weights.

Implemented iteration follows.

```python
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline

import IPython.display as ipd  # for display and clear_output
import time  # for sleep
```

```python
# Make some training data
n = 20
X = np.linspace(0.,20.0,n).reshape((n,1)) - 10
T = 0.2 + 0.05 * (X+10) + 0.4 * np.sin(X+10) + 0.2 * np.random.normal(size=(n,1))

# Make some testing data
Xtest = X + 0.1*np.random.normal(size=(n,1))
Ttest = 0.2 + 0.05 * (Xtest+10) + 0.4 * np.sin(Xtest+10) + 0.2 * np.random.normal(size=(n,1))

nSamples = X.shape[0]
nOutputs = T.shape[1]
```

```python
# Set parameters of neural network
nHiddens = 10

rhoh = 0.5
rhoo = 0.1

rh = rhoh / (nSamples*nOutputs)
ro = rhoo / (nSamples*nOutputs)

# Initialize weights to uniformly distributed values between small normally-distributed between -0.1 and 0.1
V = 0.1*2*(np.random.uniform(size=(1+1,nHiddens))-0.5)
W = 0.1*2*(np.random.uniform(size=(1+nHiddens,nOutputs))-0.5)

# Add constant column of 1's
def addOnes(A):
    return np.insert(A, 0, 1, axis=1)
X1 = addOnes(X)
Xtest1 = addOnes(Xtest)

# Take nReps steepest descent steps in gradient descent search in mean-squared-error function
nReps = 30000
# collect training and testing errors for plotting
errorTrace = np.zeros((nReps,2))

fig = plt.figure(figsize=(10,8))
for reps in range(nReps):

    # Forward pass on training data
    Z = np.tanh(X1 @ V)
    Z1 = addOnes(Z)
    Y = Z1 @ W

    # Error in output
    error = T - Y

    # TODO: Backward pass - the backpropagation and weight update steps

    # error traces for plotting
    errorTrace[reps,0] = np.sqrt(np.mean((error**2)))
    Ytest = addOnes(np.tanh(Xtest1 @ V)) @ W  #!! Forward pass in one line
    errorTrace[reps,1] = np.sqrt(np.mean((Ytest-Ttest)**2))

    if reps % 1000 == 0 or reps == nReps-1:
        plt.clf()
        plt.subplot(3,1,1)
        plt.plot(errorTrace[:reps,:])
        plt.ylim(0,0.7)
        plt.xlabel('Epochs')
        plt.ylabel('RMSE')
        plt.legend(('Train','Test'),loc='upper left')

        plt.subplot(3,1,2)
        plt.plot(X,T,'o-',Xtest,Ttest,'o-',Xtest,Ytest,'o-')
        plt.xlim(-10,10)
        plt.legend(('Training','Testing','Model'),loc='upper left')
        plt.xlabel('$x$')
        plt.ylabel('Actual and Predicted $f(x)$')

        plt.subplot(3,1,3)
        plt.plot(X,Z)
        plt.ylim(-1.1,1.1)
        plt.xlabel('$x$')
        plt.ylabel('Hidden Unit Outputs ($z$)');

        ipd.clear_output(wait=True)
        ipd.display(fig)
ipd.clear_output(wait=True)
```

```python

```

$\newcommand{\xv}{\mathbf{x}}
 \newcommand{\wv}{\mathbf{w}}
 \newcommand{\yv}{\mathbf{y}}
 \newcommand{\zv}{\mathbf{z}}
 \newcommand{\av}{\mathbf{a}}
 \newcommand{\Chi}{\mathcal{X}}
 \newcommand{\R}{\rm I\!R}
 \newcommand{\sign}{\text{sign}}
 \newcommand{\Tm}{\mathbf{T}}
 \newcommand{\Xm}{\mathbf{X}}
 \newcommand{\Xlm}{\mathbf{X1}}
 \newcommand{\Wm}{\mathbf{W}}
 \newcommand{\Vm}{\mathbf{V}}
 \newcommand{\Ym}{\mathbf{Y}}
 \newcommand{\Zm}{\mathbf{Z}}
 \newcommand{\Zlm}{\mathbf{Z1}}
 \newcommand{\I}{\mathbf{I}}
 \newcommand{\muv}{\boldsymbol\mu}
 \newcommand{\Sigmav}{\boldsymbol\Sigma}
 \newcommand{\Phiv}{\boldsymbol\Phi}
$

# Optimization

So far, we have been using gradient descent to find minimum or maximum values in our error function.
In general, we call this maximization or minimization problem as an **optimization problem**.

In optimization problems, we look for the largest or the smallest value that a function can take. By systematically choosing input vales within the constraint set, optimization problem seeks for the best available values of an objective function.

So, for a given function $f(x)$ that maps $f: \Xm \rightarrow \Ym $ where $\Ym \subset \R$,
we are looking for a $x^* \in \Xm$ that satisfies

$$
\begin{cases}
    f(x^*) \le f(x) &\forall x       & \quad \text{if } \text{ minimization}\\
    f(x^*) \ge f(x) &\forall x  & \quad \text{if } \text{ maximization}.
  \end{cases}
$$

The optimization problems are often expressed in following notation.

$$
\begin{equation*}
\begin{aligned}
& \underset{x}{\text{minimize}}
& & f(x) \\
& \text{subject to}
& & x \le b_i, \; i = 1, \ldots, m,\\
 &&& x \ge 0.
\end{aligned}
\end{equation*}
$$

## Least Squares

$$
\begin{equation*}
\begin{aligned}
& \underset{\wv}{\text{minimize}}
& & \Vert \Xm \wv - t\Vert^2
\end{aligned}
\end{equation*}
$$

As we discussed, least-squares problems can be solved analytically, $\wv = (\Xm^\top \Xm)^{-1} \Xm^\top t$.
We easily formulate least-sqaures and solve very efficiently.

## Linear Programming

$$
\begin{equation*}
\begin{aligned}
& \underset{\xv}{\text{minimize}}
& & \wv^\top \xv \\
& \text{subject to}
& & \av_i^\top \xv \le b_i, \; i = 1, \ldots, m.
\end{aligned}
\end{equation*}
$$

Linear programming or linear optimization finds a maximum or minimum from a mathematical model that is represented by linear relationships.
There is no analytical formular for a solution, but there are reliable algorithms that solve LP efficiently.

## Convex Optimization

$$
\begin{equation*}
\begin{aligned}
& \underset{x}{\text{minimize}}
& & f_0(x) \\
& \text{subject to}
& & f_i(x) \leq b_i, \; i = 1, \ldots, m.
\end{aligned}
\end{equation*}
$$

Convex condition:
$$
f_i(\alpha x_1 + (1-\alpha) x_2) \le \alpha f_i(x_1) + (1-\alpha) f_i(x_2)
$$

Convex optimization generalizes the linear programming problems. A convex optimization problem has the constraint set that forms convex functions. As a general model of LP, convex optimization problems do not have analytical solution but they also have reliable and efficient algorithms for it. Thus, it can be solved very quickly and reliably up to very large problems.
However, it is difficulty to recognize if it is convex or not.

## Nonlinear Optimization

For non-convex problems, we can apply local optimization methods, which is called nonlinear programming.
Starting from initial guess, it searchs for a minimal point near neighborhood. It can be fast and can be applicable large problems. However, there is no guarantee for discovery of global optimum.

## Newton's method

Newton's method approximates the curve with quadratic function repeatedly to find a temporary point or stationary point of $f$.

If we assume that for each measurement point $x^{(k)}$, we can compute $f(x^{(k)})$, $f^{\prime}(x^{(k)})$, and $f^{\prime\prime}(x^{(k)})$.
Using second order Taylor expansion, we can approximate $q(x)$ for $f(x + \Delta x)$:

$$
q(x) = f(x^{(k)}) + f^{\prime}(x^{(k)}) \Delta x + \frac{1}{2} f^{\prime\prime}(x^{(k)}) \Delta x^2
$$

where $\Delta x = (x - x^{(k)})$.

Minimizing this quadratic function,

$$
0 = q^\prime(x) = f^{\prime}(x^{(k)}) + f^{\prime\prime}(x^{(k)}) \Delta x.
$$

Setting $x = x^{(k+1)}$, we can get

$$
 x^{(k+1)} =  x^{(k)} - \frac{f^{\prime}(x^{(k)})}{f^{\prime\prime}(x^{(k)})}.
$$

```python
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
import scipy.optimize as opt
from scipy.optimize import rosen, minimize

```

```python
# examples are from http://people.duke.edu/~ccc14/sta-663-2017/14C_Optimization_In_Python.html

x = np.linspace(-5, 5, 1000)
y = np.linspace(-5, 5, 1000)

xs, ys = np.meshgrid(x, y)
zs = rosen(np.vstack([xs.ravel(), ys.ravel()])).reshape(xs.shape)

plt.figure(figsize=(8,8))
plt.contour(xs, ys, zs, np.arange(10)**5, cmap='jet')
plt.text(1, 1, 'x', va='center', ha='center', color='k', fontsize=30);
```

```python
from scipy.optimize import rosen_der, rosen_hess

def reporter(p):
    """record the points visited"""
    global ps
    ps.append(p)

# starting position
x0 = np.array([4,-4.1])

ps = [x0]
minimize(rosen, x0, method="Newton-CG", jac=rosen_der, hess=rosen_hess, callback=reporter)

```

         fun: 1.3642782750354208e-13
         jac: array([  1.21204353e-04,  -6.08502470e-05])
     message: 'Optimization terminated successfully.'
        nfev: 38
        nhev: 26
         nit: 26
        njev: 63
      status: 0
     success: True
           x: array([ 0.99999963,  0.99999926])

```python
ps = np.array(ps)
plt.figure(figsize=(16, 8))
plt.subplot(121)
plt.contour(xs, ys, zs, np.arange(10)**5, cmap='jet')
plt.plot(ps[:, 0], ps[:, 1], '-ro')
plt.subplot(122)
plt.semilogy(range(len(ps)), rosen(ps.T));
```

## Vs. others?

Now, let us take a look at other optimization tools including naive steepest descent and scaled conjugate gradient ([Moller, 1997](http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.50.8063&rep=rep1&type=pdf)).
To run this properly, you need to download [grad.py](https://webpages.uncc.edu/mlee173/teach/itcs4156online/notes/grad.py) under your current work folder.

```python
from grad import steepest

res = steepest(np.array(x0), rosen_der, rosen, stepsize=0.0001, wtracep=True, ftracep=True)

ps = np.array(res['wtrace'])
plt.figure(figsize=(16, 8))
plt.subplot(121)
plt.contour(xs, ys, zs, np.arange(10)**5, cmap='jet')
plt.plot(ps[:, 0], ps[:, 1], '-ro')
plt.subplot(122)
plt.semilogy(range(len(ps)), res['ftrace']);
```

```python
from grad import scg

res = scg(np.array(x0), rosen_der, rosen, wtracep=True, ftracep=True)
res1 = scg(np.array([-4, 4]), rosen_der, rosen, wtracep=True, ftracep=True)

ps = np.array(res['wtrace'])
ps1 = np.array(res1['wtrace'])
plt.figure(figsize=(16, 8))
plt.subplot(121)
plt.contour(xs, ys, zs, np.arabnge(10)**5, cmap='jet')
plt.plot(ps[:, 0], ps[:, 1], '-ro')
plt.plot(ps1[:, 0], ps1[:, 1], '-bo')
plt.subplot(122)
plt.semilogy(range(len(ps)), res['ftrace']);
```

```python
x0 = [-4, 4]
ps = [x0]
minimize(rosen, x0, method="Newton-CG", jac=rosen_der, hess=rosen_hess, callback=reporter)

ps = np.array(ps)
plt.figure(figsize=(16, 8))
plt.subplot(121)
plt.contour(xs, ys, zs, np.arange(10)**5, cmap='jet')
plt.plot(ps[:, 0], ps[:, 1], '-ro')
plt.subplot(122)
plt.semilogy(range(len(ps)), rosen(ps.T));
```

# Neural Network!

Now, let us use this optimization trick for our neural networks.

```python
# standardization class
class Standardizer:
    """ class version of standardization """
    def __init__(self, X, explore=False):
        self._mu = np.mean(X,0)
        self._sigma = np.std(X,0)
        if explore:
            print ("mean: ", self._mu)
            print ("sigma: ", self._sigma)
            print ("min: ", np.min(X,0))
            print ("max: ", np.max(X,0))

    def set_sigma(self, s):
        self._sigma[:] = s

    def standardize(self,X):
        return (X - self._mu) / self._sigma

    def unstandardize(self,X):
        return (X * self._sigma) + self._mu

```

```python

""" Neural Network
        referenced NN code by Chuck Anderson in R and C++

                                by Jake Lee (lemin)

    example usage:
        X = numpy.array([0,0,1,0,0,1,1,1]).reshape(4,2)
        T = numpy.array([0,1,1,0,1,0,0,1]).reshape(4,2)

        nn = nnet.NeuralNet([2,3,2])
        nn.train(X,T, wprecision=1e-20, fprecision=1e-2)
        Y = nn.use(X)

"""
from grad import scg, steepest
from copy import copy

class NeuralNet:
    """ neural network class for regression

        Parameters
        ----------
        nunits: list
            the number of inputs, hidden units, and outputs

        Methods
        -------
        set_hunit
            update/initiate weights

        pack
            pack multiple weights of each layer into one vector

        forward
            forward processing of neural network

        backward
            back-propagation of neural network

        train
            train the neural network

        use
            appply the trained network for prediction

        Attributes
        ----------
        _nLayers
            the number of hidden unit layers

        rho
            learning rate

        _W
            weights
        _weights
            weights in one dimension (_W is referencing _weight)

        stdX
            standardization class for data
        stdT
            standardization class for target

        Notes
        -----

    """

    # TODO: Try to implement Neural Network class with the member variables and methods described above

```

```python
X = np.array([0,0,1,0,0,1,1,1]).reshape(4,2)
T = np.array([0,1,1,0,1,0,0,1]).reshape(4,2)

nn = NeuralNet([2,3,2])
nn.train(X, T)
Y = nn.use(X)
```

```python
Y
```

    array([[  6.57854210e-07,   1.00000047e+00],
           [  1.00000100e+00,  -1.17983002e-07],
           [  1.00000194e+00,  -2.09443329e-07],
           [  2.12270757e-06,   1.00000155e+00]])

```python
T
```

    array([[0, 1],
           [1, 0],
           [1, 0],
           [0, 1]])

```python
X
```

    array([[0, 0],
           [1, 0],
           [0, 1],
           [1, 1]])

```python
# repeating the previous example

# Make some training data
n = 20
X = np.linspace(0.,20.0,n).reshape((n,1)) - 10
T = 0.2 + 0.05 * (X+10) + 0.4 * np.sin(X+10) + 0.2 * np.random.normal(size=(n,1))

# Make some testing data
Xtest = X + 0.1*np.random.normal(size=(n,1))
Ttest = 0.2 + 0.05 * (Xtest+10) + 0.4 * np.sin(Xtest+10) + 0.2 * np.random.normal(size=(n,1))

nSamples = X.shape[0]
nOutputs = T.shape[1]
```

```python
nn = NeuralNet([1,3,1])
nn.train(X, T, ftracep=True)
Ytest, Z = nn.use(Xtest, retZ=True)

plt.figure(figsize=(10,8))
plt.subplot(3,1,1)
plt.plot(nn.ftrace)
plt.ylim(0,0.7)
plt.xlabel('Epochs')
plt.ylabel('RMSE')
plt.legend(('Train','Test'),loc='upper left')

plt.subplot(3,1,2)
plt.plot(X,T,'o-',Xtest,Ttest,'o-',Xtest,Ytest,'o-')
plt.xlim(-10,10)
plt.legend(('Training','Testing','Model'),loc='upper left')
plt.xlabel('$x$')
plt.ylabel('Actual and Predicted $f(x)$')

plt.subplot(3,1,3)
plt.plot(X, Z[1])
plt.ylim(-1.1,1.1)
plt.xlabel('$x$')
plt.ylabel('Hidden Unit Outputs ($z$)');
```