## setup and notebook configuration

```python
# scientific python stack
import numpy as np
import scipy as sp
import sympy as sym
import orthopy, quadpy
```

```python
# matplotlib, plotting setup
import matplotlib as mpl
import matplotlib.pyplot as plt
import matplotlib.tri as mtri               # delaunay triangulation
from mpl_toolkits.mplot3d import Axes3D     # surface plotting
import seaborn as sns                       # nice plotting defaults
import cmocean as cmo                       # ocean colormaps
sym.init_printing(use_latex='mathjax')
sns.set()
%matplotlib inline
%config InlineBackend.figure_format = 'svg'
```

    /Users/corbin/virtual_envs/FEMexperiment_env/lib/python3.6/site-packages/cmocean/tools.py:76: MatplotlibDeprecationWarning: The is_string_like function was deprecated in version 2.1.
      if not mpl.cbook.is_string_like(rgbin[0]):

```python
%load_ext autoreload
%autoreload 2
```

```python
# local imports
import src.fem_base.master.mk_basis_nodal as mbn
import src.fem_base.master.mk_master as mkm
import src.fem_maps.fem_map as fem_map
```

# creation of 1D nodal bases

We define the master 1D element as $I\in[-1, 1]$

```python
b = mbn.Basis_nodal(order=1, dim=1, element=0)
b.plot_elm()
```

```python
fig, ax = plt.subplots(1, 5, figsize = (10, 1))
xx = np.linspace(-1, 1, 100)
pts = [[x, 0, 0] for x in xx]

for order in range(5):
    b = mbn.Basis_nodal(order=order, dim=1, element=0)
    yy = b.eval_at_pts(pts=pts)
    for basis_fn in range(b.nb):
        ax[order].plot(xx, yy[:, basis_fn])
    ax[order].set_title(r'$p = {}$'.format(order))
```

## construction of vandermonde matrices

```python
x = np.linspace(-1, 1, 100)
n_polys = 4
vals = orthopy.line_segment.tree_jacobi(x, n=n_polys, alpha=0, beta=0, standardization='normal')
for i in range(n_polys):
    plt.plot(x, vals[i], label='P_{}'.format(i))
plt.legend()

plt.title('normalized Legendre polynomials')
plt.show()
```

These polynomials agree with the explicitly listed polynomials in Hesthaven, so we know that they are orthonormalized correctly.

```python
def Jacobi_Poly(r, alpha, beta, N):
    """ wraps orthopy to return Jacobi polynomial """
    return orthopy.line_segment.tree_jacobi(r, n=N-1, alpha=alpha, beta=beta, standardization='normal')

def P_tilde(r, N):
    P = np.zeros((len(r), N))
    polyvals = Jacobi_Poly(r, alpha=0, beta=0, N=N)
    for j in range(N):
        P[:, j] = polyvals[j]
    return P.T

def Vandermonde1D(N, x):
    """ initialize 1D vandermonde Matrix Vij = phi_j(x_i)"""
    V1D = np.zeros((len(x), N))
    JacobiP = Jacobi_Poly(x, alpha=0, beta=0, N=N)
    for j, polyvals in enumerate(JacobiP):
        V1D[:, j] = polyvals
    return V1D
```

```python
def LegendreGaussLobatto(N):
    GL = quadpy.line_segment.GaussLobatto(N, a=0., b=0.)
    return GL.points, GL.weights

def GaussLegendre(N):
    GL = quadpy.line_segment.GaussLegendre(N)
    return GL.points, GL.weights
```

An important conceptual point is that the Vandermonde matrix here is NOT the shape function matrix, it's the Vandermonde matrix of the Orthonormal polynomial basis. We will see this later as we have to create the shape function matrices.

## properties / conditioning of vandermonde matrices

```python
equi_det, LGL_det = [], []
for N in range(2, 35):
    nb = N + 1
    equi_pts = np.linspace(-1, 1, nb)
    V = Vandermonde1D(nb, equi_pts)
    equi_det.append(np.linalg.det(V))

    LGL_pts, _ = LegendreGaussLobatto(nb)
    V = Vandermonde1D(nb, LGL_pts)
    LGL_det.append(np.linalg.det(V))
```

```python
plt.semilogy(list(range(2, 35)), equi_det, label='equidistant')
plt.semilogy(list(range(2, 35)), LGL_det, label='LGL nodes')
plt.legend()
plt.show()
```

This result agrees with Hesthaven.

```python
# construct generic lagrange interpolant
from scipy.interpolate import lagrange

def lagrange_polys(pts):
    lagrange_polys = []
    for i, pt in enumerate(pts):
        data = np.zeros_like(pts)
        data[i] = 1
        lagrange_polys.append(lagrange(pts, data))
    return lagrange_polys

def lagrange_basis_at_pts(lagrange_polys, eval_pts):
    """ evaluates lagrange polynomials at eval_pts"""
    result = np.zeros((len(lagrange_polys) ,len(eval_pts)))
    for i, poly in enumerate(lagrange_polys):
        result[i, :] = lagrange_polys[i](eval_pts)
    return result
```

plot lagrange polys over equally spaced vs LGL points

```python
N = 5
lp = np.linspace(-1, 1, N)
lpolys = lagrange_polys(lp)

vN = 100
view_pts = np.linspace(-1, 1, vN)
li = lagrange_basis_at_pts(lpolys, view_pts)
plt.plot(view_pts, li.T)
plt.title('lagrange polynomials over equally spaced points')
plt.show()
```

```python
N = 5
lp, _ = LegendreGaussLobatto(N)
lpolys = lagrange_polys(lp)

vN = 100
view_pts = np.linspace(-1, 1, vN)
li = lagrange_basis_at_pts(lpolys, view_pts)
plt.plot(view_pts, li.T)
plt.title('lagrange polynomials over LGL points')
plt.show()
```

So beautiful! By moving the lagrange data points to the nodal points, our basis functions don't exceed 1, unlike in the above plot, where we are already seeing a slight Runge phenomenon.

## vandermonde relations

### relationship between vandermonde $V$, basis polynomials $\tilde{\mathbf{P}}$, and lagrange basis functions (shape functions) $\ell$

Hesthaven makes the claim that $V^T \mathbf{\ell}(r) = \tilde{\mathbf{P}}(r)$ in (3.3).

In Hesthaven's notation, $N$ denotes the polynomial order, $N_p$ denotes the number of nodal points (we would call $nb$), and let's call the number of "view points" `xx`, which are arbitrary.

Then the shapes of the Hesthaven structures are:
- $\mathbf{\ell}$, $\tilde{\mathbf{P}}$, $V$ are all (`nb`, `xx`)
- $V^T \ell$ is (`xx`, `nb`) x (`nb`, `xx`) $\rightarrow$ (`xx`, `xx`) where rows contain the values of polynomials $\tilde{\mathbf{P}}$

This works for either equidistant points or the LGL points.

```python
N = 5
lp, _ = LegendreGaussLobatto(N)
#lp = np.linspace(-1, 1, N)
view_pts = np.linspace(-1, 1, 50)

l_polys = lagrange_polys(pts=lp)
ℓ = lagrange_basis_at_pts(l_polys, eval_pts=view_pts)
V = Vandermonde1D(N=len(view_pts), x=lp)
P = np.dot(V.T, ℓ)

# plot the result
plt.plot(view_pts, ℓ.T, '--')
plt.plot(view_pts, P[0:3,:].T)
plt.show()
```

We see that indeed we recover the Legendre polynomials.

More directly, we can invert the relation to find that

$$\ell = (V^T)^{-1} \tilde{\mathbf{P}}$$

which allows us to create our nodal shape functions.

```python
nb = 4
nodal_pts, _ = LegendreGaussLobatto(nb)
view_pts = np.linspace(-1, 1, 50)

# create the Vandermonde, P matrices
V = Vandermonde1D(N=nb, x=nodal_pts)
Vti = np.linalg.inv(V.T)
P = P_tilde(r=view_pts, N=nb)
print('shape of Vandermonde: {}'.format(V.shape))
print('shape of P: {}'.format(P.shape))

yy = np.dot(Vti, P)
plt.plot(view_pts, yy.T)
plt.title('nodal shape functions generated from orthogonal basis polynomials')
plt.show()
```

    shape of Vandermonde: (4, 4)
    shape of P: (4, 50)

### relationship between vandermonde $V$ and mass matrix

We can build on the relationship developed in the section above to form the mass matrix for a nodal basis. We note that

$M_{ij} = \int_{-1}^{1}\ell_i(r)\, \ell_j(r) \,dr = (\ell_i, \ell_j)_I$, and if we expand out $\ell = (V^T)^{-1}\tilde{\mathbf{P}}$, it turns out (page 51)

$$M = (V V^T)^{-1}$$

because of the orthogonal nature of our choice of basis function; the implication is that we can compute the integrals over the master element without the explicit need for quadrature points or weights. Note first that $\phi_i(\xi) = \sum_{n=1}^{nb} (V^T)_{in}^{-1} \tilde{P}_{n-1}(\xi)$. Then

\begin{align}
M_{ij} &= \int^{1}_{-1} \phi_i(\xi)\,\phi_j(\xi)\,d\xi
      = \int^{1}_{-1}\left[\sum_{k=1}^{nb} (V^T)_{ik}^{-1} \tilde{P}_{k-1}(\xi)
      \sum_{m=1}^{nb} (V^T)_{jm}^{-1} \tilde{P}_{m-1}(\xi) \right]\, d\xi \\
      &= \sum_{k=1}^{nb} \sum_{m=1}^{nb} (V^T)_{ik}^{-1}
       (V^T)_{jm}^{-1} \int^{1}_{-1}\tilde{P}_{k-1}(\xi) \tilde{P}_{m-1}(\xi)
       =\sum_{k=1}^{nb} \sum_{m=1}^{nb} (V^T)_{ik}^{-1}
       (V^T)_{jm}^{-1} \delta_{km} \\
       &=\sum_{k=1}^{nb} (V^T)_{im}^{-1}
       (V^T)_{jm}^{-1} = \sum_{k=1}^{nb} (V^T)_{mi}^{-1} (V)_{mj}^{-1} \\
       &= (V^{T})^{-1} V^{-1} = (VV^T)^{-1}
\end{align}

Where note we've used the cute trick that $\int_{-1}^1 \tilde{P}_m \tilde{P}_n = \delta_{mn}$, since we chose an __orthonormal__ modal basis. Orthogonal wouldn't have done it, but an orthonormal modal basis has this property.

We can check this relation against the more traditional way of constructing the mass matrix with quadrature. `master.shap` has dimensions of (`n_quad`, `nb`)

```python
order = 3
m1d = mkm.Master_nodal(order=order, dim=1, element=0)
xq, wq = m1d.cube_pts, m1d.cube_wghts
shap = m1d.shap
shapw = np.dot(np.diag(wq), m1d.shap)
M_quadrature = np.dot(shap.T, shapw)

Np = order + 1
nodal_points, _ = LegendreGaussLobatto(Np)
V = Vandermonde1D(N=Np, x=nodal_points)
M_vand = np.linalg.inv(np.dot(V, V.T))

# this will throw an error if not correct
assert(np.allclose(M_quadrature, M_vand))
```

## efficient computation of derivatives of the basis functions

### derivatives of Legendre polynomials

In order to compute the derivatives of the shape functions (which are expressed via the vandermonde matrix $V$), we must take the derivatives with respect to the orthogonal basis polynomials. There is an identity (Hesthaven, p. 52)

$$ \frac{d \tilde{P}_n}{d r} = \sqrt{n(n+1)}\,\tilde{P}^{(1,1)}_{n-1}$$

This is in contrast to directly differentiating either the coefficients of $\tilde{P}$ or more directly the nodal shape functions $\ell$ if the explicit polynomial form is known (like in `scipy`, but this becomes trickier in multiple dimensions). As it turns out, the first approach is a very efficient way to compute these operators.

```python
def Jacobi_Poly_Derivative(r, alpha, beta, N):
    """ take a derivative of Jacobi Poly, more general than above
    copy the format of orthopy (list of arrays)
    """
    dp = [np.zeros_like(r)]
    Jacobi_P = Jacobi_Poly(r, alpha + 1, beta + 1, N)
    for n in range(1, N+1):
        gamma = np.sqrt(n * (n + alpha + beta + 1))
        dp.append(gamma * Jacobi_P[n-1])
    return dp

#def dP_tilde(r, N):
#    P = np.zeros((len(r), N))
#    polyvals = Jacobi_Poly_Derivative(r, alpha=0, beta=0, N=N)
#    for j in range(N):
#        P[:, j] = polyvals[j]
#    return P
```

We can examine some of the derivatives of the Legendre polynomials.

```python
# some unit testing
# first jacobi poly is const, so der should be 0
xx = np.linspace(-1, 1, 50)
jpd = Jacobi_Poly_Derivative(xx, alpha=0, beta=0, N=3)
for i, polyder in enumerate(jpd):
    plt.plot(xx, polyder, label=r'$P_{}^\prime(x)$'.format(i))
plt.legend()
plt.show()
```

These look good. The derivative of the first Legendre polynomial is analytically 0, $P_1^\prime = \sqrt{3/2}$, $P_2^\prime$ should be linear, $P_3^\prime$ should be quadratic.

### discrete derivative operators

We can declare the derivative Vandermonde matrix, and invert it in the same manner to obtain the derivatives of the nodal shape functions.

This works because
$$V^T \ell = P \Rightarrow V^T \frac{d}{dx} \ell = \frac{d}{dx}P$$

Hence $$V_r\equiv V^T D_r^T, \qquad {V_r}_{(ij)} = \frac{d \tilde{P}_j(r_i)}{d x} $$

and finally $D_r = V_r V^{-1}$ (see Hesthaven, p. 53), as well as $S = M D_r $, where $S_{ij} = \left(\phi_i, \frac{d\phi_j}{dx}\right)_I$, and where $M$ is the mass matrix.

```python
def GradVandermonde1D(N, x):
    Vr = np.zeros((len(x), N))
    dJacobi_P = Jacobi_Poly_Derivative(x, alpha=0, beta=0, N=N-1)
    for j, polyder in enumerate(dJacobi_P):
        Vr[:,j] = polyder
    return Vr
```

```python
p = 3
nb = p+1
nodal_pts, _ = LegendreGaussLobatto(nb)
#nodal_pts = np.linspace(-1, 1, nb)
view_pts = np.linspace(-1, 1, 50)

# grad vandermonde
V  = Vandermonde1D(N=nb, x=nodal_pts)
Vr = GradVandermonde1D(N=nb, x=view_pts)
Vi = np.linalg.inv(V)
Dr = np.dot(Vr, Vi)

print('shape Vr:    {}'.format(Vr.shape))
print('shape V inv: {}'.format(Vi.shape))
print('shape Dr:    {}'.format(Dr.shape))

# shape functions
V = Vandermonde1D(N=nb, x=nodal_pts)
Vti = np.linalg.inv(V.T)
P = P_tilde(r=view_pts, N=nb)
shap = np.dot(Vti, P)

# shape functions at view points
fig, ax = plt.subplots(1, 2, figsize=(8, 4))
ax[0].plot(view_pts, shap.T, '--')
ax[0].set_title(r'nodal shape functions $\phi$')
ax[1].plot(view_pts, Dr)
ax[1].set_title(r'derivatives of nodal shape functions $\frac{d \phi}{dx}$')
plt.show()
```

    shape Vr:    (50, 4)
    shape V inv: (4, 4)
    shape Dr:    (50, 4)

As a remark, we can once again show the effect of using Legendre Gauss Lobatto points vs equally spaced nodal points.

```python
N = 8
nb = N+1
nodal_pts_LGL, _ = LegendreGaussLobatto(nb)
nodal_pts_equi = np.linspace(-1, 1, nb)
view_pts = np.linspace(-1, 1, 100)

# shape functions at view points
fig, ax = plt.subplots(figsize=(8, 8))
ax.set_yticks([])
ax.set_xticks([])
labels = ['LGL nodes', 'uniform nodes']
for idx, nodal_pts in enumerate([nodal_pts_LGL, nodal_pts_equi]):
    # grad vandermonde
    V  = Vandermonde1D(N=nb, x=nodal_pts)
    Vr = GradVandermonde1D(N=nb, x=view_pts)
    Vi = np.linalg.inv(V)
    Dr = np.dot(Vr, Vi)

    # shape functions
    V = Vandermonde1D(N=nb, x=nodal_pts)
    Vti = np.linalg.inv(V.T)
    P = P_tilde(r=view_pts, N=nb)
    shap = np.dot(Vti, P)

    # plot
    ax = fig.add_subplot(2, 2, idx*2+1)
    ax.plot(view_pts, shap.T)
    ax.set_yticks([0, 1])
    ax.set_title(r' $\phi$, {}'.format(labels[idx]))
    ax = fig.add_subplot(2, 2, idx*2+2)
    ax.plot(view_pts, Dr)
    ax.set_title(r'$\f{}$, {}'.format('rac{d \phi}{dx}',labels[idx]))
plt.subplots_adjust(wspace=0.2, hspace=0.2)
fig.suptitle(r'$\phi$ and $\frac{d\phi}{d x}$, LGL vs uniformly-spaced nodes')
plt.show()
```

### remarks on discrete derivative operators

Suppose we compute the derivative matrix $D_r$ at the nodal points for some order $p$. There are some interesting properties to understand about these derivative matrices.

#### annihilation of constant vectors

Note that if we represent a function nodally, i.e., $u = c_i \phi_i(x)$, then

$$ \frac{du}{dx} = \frac{d}{dx}(c_i \phi_i(x)) = c_i \frac{d\phi_i}{dx} $$

Therefore, if we want to discretely take a derivative of a function (we'll represent the function on the master element for now, but trivially, we could map it to some other region), it suffices to multiply the derivative operator with the nodal vector $D_r u$. It should be clear, then, that $D_r$ will annihilate any constant vector.

```python
p = 2
nb = p+1
nodal_pts, _ = LegendreGaussLobatto(nb)
u = np.ones_like(nodal_pts)

# grad vandermonde
V  = Vandermonde1D(N=nb, x=nodal_pts)
Vr = GradVandermonde1D(N=nb, x=nodal_pts)
Vi = np.linalg.inv(V)
Dr = np.dot(Vr, Vi)

duh = np.dot(Dr, u)
print(np.max(np.abs(duh)))
```

    1.1102230246251565e-16

#### exponential convergence in $p$ for smooth functions

If the function of which we are attempting to take the discrete derivative is smooth (in the sense of infinitely differentiable), then we will see exponential convergence to the analytical solution w/r/t polynomial order of the nodal basis.

```python
ps = [1, 2, 3, 4, 8, 12, 16, 18, 24, 32]
errs = []
fig, ax = plt.subplots(1, 3, figsize=(8, 3))
for p in ps:
    nb = p+1
    nodal_pts, _ = LegendreGaussLobatto(nb)
    view_pts = np.linspace(-1, 1, 100)

    # grad vandermonde
    V  = Vandermonde1D(N=nb, x=nodal_pts)
    Vr = GradVandermonde1D(N=nb, x=nodal_pts)
    Vi = np.linalg.inv(V)
    Dr = np.dot(Vr, Vi)

    # nodal shap
    V = Vandermonde1D(N=nb, x=nodal_pts)
    Vti = np.linalg.inv(V.T)
    P = P_tilde(r=view_pts, N=nb)
    view_shap = np.dot(Vti, P)

    u = np.sin(nodal_pts-np.pi/4.)
    du = np.cos(view_pts-np.pi/4)
    duh = np.dot(Dr, u)

    view_duh = np.dot(duh, view_shap)
    err = np.max(np.abs(view_duh - du))
    errs.append(err)

    # plot first few
    if p < 4:
        ax[p-1].plot(view_pts, np.sin(view_pts), label=r'$u$')
        ax[p-1].plot(view_pts, du, label=r'$u^\prime$')
        ax[p-1].plot(view_pts, view_duh, '--', label=r'$du_h$')
        ax[p-1].set_title(r'$p={}$'.format(p))
        ax[p-1].legend()
plt.show()
```

```python
fig, ax = plt.subplots()
ax.semilogy(ps, errs)
ax.set_xticks(ps)
ax.set_ylabel(r'$||du - du_h||_{L_\infty}$')
ax.set_xlabel('polynomial order p')
ax.set_title('exponential convergence of discrete derivative')
plt.show()
```

# creation of a 1D master element

Define a simple nodal basis object for 1D problems -- no need to be fancy, pretty much the only thing we need this for is to get nodal shape functions and their derivatives efficiently. The underlying orthonormal Legendre polynomial basis is hidden to the user.

```python
class NodalBasis1D(object):
    """ minimalist nodal basis object:
    efficiently computes shape functions and their derivatives
    """
    def __init__(self, p, node_spacing='GAUSS_LOBATTO'):
        self.nb = p + 1
        if node_spacing == 'GAUSS_LOBATTO':
            self.nodal_pts, _ = LegendreGaussLobatto(self.nb)
        elif node_spacing == 'EQUIDISTANT':
            self.nodal_pts = np.linspace(-1, 1, self.nb)
        else: raise ValueError('node_spacing {} not recognized'.format(node_spacing))

    def shape_functions_at_pts(self, pts):
        """ computes shape functions evaluated at pts on [-1, 1]
        @retval shap (len(pts), nb) phi_j(pts[i])
        """
        V = Vandermonde1D(N=self.nb, x=self.nodal_pts)
        VTinv = np.linalg.inv(V.T)
        P = P_tilde(r=pts, N=self.nb)
        shap = np.dot(VTinv, P)
        return shap.T

    def shape_function_derivatives_at_pts(self, pts):
        """ computes shape function derivatives w/r/t x on [-1, 1]
        @retval shap_der, (Dr in Hesthaven), (len(pts), nb) d/dx phi_j(pts[i])
        """
        V  = Vandermonde1D(N=self.nb, x=self.nodal_pts)
        Vx = GradVandermonde1D(N=self.nb, x=pts)
        Vinv = np.linalg.inv(V)
        shap_der = np.dot(Vx, Vinv)
        return shap_der
```

Define a 1D master element, which is built on top of the 1D basis.
- Precompute shape functions at the nodal points and Gauss Legendre quadrature points, both are useful for different types of schemes. We use Gauss Legendre points instead of Gauss Lobatto points because they can integrate degree $2n-1$ polynomials exactly instead of $2n - 3$, where $n$ is the number of integration points. We would like to integrate $(\phi_i, \phi_j)_{\hat{K}}$, which is order 2$p$, so to integrate the mass matrix exactly, we need $2p + 1$ points, and common practice is $2p+2$. Since quadrature in 1D is cheap, we opt for the latter.
- Precompute mass matrix $M_{ij} = (\phi_i, \phi_j)$ and stiffness matrices $S_{ij} = \left(\phi_i, \frac{d\phi_j}{dx}\right)$, $K_{ij} = \left(\frac{d\phi_i}{dx}, \frac{d\phi_j}{dx}\right)$. Additionally, store $M^{-1}$, as it is commonly used. Although Hesthaven's method for mass and stiffness matrices are elegant, they rely on the underlying choice of an orthanormal modal basis. Since this class could be overloaded to work with other choices of basis, better to simply compute these matrices with quadrature.

```python
class Master1D(object):
    """ minimalist 1D master object """
    def __init__(self, p, nquad_pts=None, *args, **kwargs):
        self.p, self.nb = p, p+1
        self.basis = NodalBasis1D(p=p, **kwargs)
        self.nodal_pts = self.basis.nodal_pts
        self.nq = 2*self.p + 2 if nquad_pts is None else nquad_pts
        self.quad_pts, self.wghts = GaussLegendre(self.nq)

        # shape functions at nodal and quadrature points
        self.shap_nodal, self.dshap_nodal = self.mk_shap_and_dshap_at_pts(self.nodal_pts)
        self.shap_quad,  self.dshap_quad = self.mk_shap_and_dshap_at_pts(self.quad_pts)

        # mass, stiffness matrices
        self.M, self.S, self.K = self.mk_M(), self.mk_S(), self.mk_K()
        self.Minv = np.linalg.inv(self.M)

        # lifting permuatation matrix L (0s, 1s)
        self.L = self.mk_L()

    def mk_shap_and_dshap_at_pts(self, pts):
        shap = self.basis.shape_functions_at_pts(pts)
        dshap = self.basis.shape_function_derivatives_at_pts(pts)
        return shap, dshap

    def mk_M(self):
        shapw = np.dot(np.diag(self.wghts), self.shap_quad)
        M = np.dot(self.shap_quad.T, shapw)
        return M

    def mk_S(self):
        dshapw = np.dot(np.diag(self.wghts), self.dshap_quad)
        S = np.dot(self.shap_quad.T, dshapw)
        return S

    def mk_K(self):
        dshapw = np.dot(np.diag(self.wghts), self.dshap_quad)
        K = np.dot(self.dshap_quad.T, dshapw)
        return K

    def mk_L(self):
        L = np.zeros((self.nb, 2))
        L[0, 0]  = 1
        L[-1, 1] = 1
        return L

    @property
    def shap_der(self):
        """ return the shape derivatives for apps expecting 2, 3D"""
        return [self.dshap_quad]
```

# creation of 1D mesh and DOF handler

## 1D mappings

For 1D problems, the mapping from the master element to physical space elements is somewhat trivial, since there's no reason for the transformation to be anything except affine. Note though, that when the 1D elements are embedded in 2D, then the transformations may be non-affine, in which case we must handle isoparametric mappings and the like. We defer this until later. For an affine mapping, we have the simple mapping

$$x(\xi) = x_L^k + \frac{1 + \xi}{2}(x_R^k - x_L^k)$$

With which we can move the nodal master points to their physical space coordinates.

```python
# build T and P arrays
P = np.linspace(2, 4, 5)
```

```python
class Mesh1D(object):
    def __init__(self, P):
        """ @param P  vertex points, sorted by x position """
        self.verts = P
        self.nElm, self.nEdges = len(self.verts) - 1, len(self.verts)
        self.connectivity = self.build_T()
        connected_one_side = np.bincount(self.connectivity.ravel()) == 1
        self.boundary_verts = np.where(connected_one_side)[0]

    def build_T(self):
        """ element connectivity array from 1D vertex list """
        T = np.zeros((self.nElm, 2), dtype=int)
        T[:,0] = np.arange(self.nElm)
        T[:,1] = np.arange(self.nElm) + 1
        return T
```

```python
class dofh_1D(object): pass

class DG_dofh_1D(dofh_1D):
    def __init__(self, mesh, master):
        self.mesh, self.master = mesh, master
        self.n_dof = self.master.nb * self.mesh.nElm
        self.dgnodes = self.mk_dgnodes()
        self.lg = self.mk_lg()
        self.lg_PM = self.mk_minus_plus_lg()
        self.nb, self.nElm = self.master.nb, self.mesh.nElm
        self.ed2elm = self.mk_ed2elm()

    def mk_dgnodes(self):
        """ map master nodal pts to element vertices def'd in self.mesh """
        dgn = np.zeros((self.master.nb, self.mesh.nElm))
        master_nodal_pts = np.squeeze(self.master.nodal_pts)
        for elm, elm_verts in enumerate(self.mesh.connectivity):
            elm_vert_pts = self.mesh.verts[elm_verts]
            elm_width = elm_vert_pts[1] - elm_vert_pts[0]
            mapped_pts = elm_vert_pts[0] + (1+master_nodal_pts)/2.*(elm_width)
            dgn[:, elm] = mapped_pts
        return dgn

    def mk_lg(self):
        """ number all dof sequentially by dgnodes """
        node_numbers = np.arange(np.size(self.dgnodes))
        lg = node_numbers.reshape(self.dgnodes.shape, order='F')
        return lg

    def mk_minus_plus_lg(self):
        """ (-) denotes element interior, (+) denotes exterior"""
        lg_PM = dict()
        lg_PM['-'] = self.lg[[0, -1], :].ravel(order='F')
        lgP = self.lg[[0, -1],:]
        lgP[0, 1: ] -= 1 # shift nodes to left of first
        lgP[1, :-1] += 1 # shift nodes to right of last
        lg_PM['+'] = lgP.ravel(order='F')
        return lg_PM

    def mk_ed2elm(self):
        """ internal map holding the indicies to reshape vector of values on faces to
        element edge space (2, nElm), duplicating the values on either side of interior faces
        """
        f2elm = np.zeros((2, self.nElm))
        faces = np.arange(self.mesh.nEdges)
        # numpy magic is doing the following:
        #
        # [[0, 1, 2, 3]
        #  [0, 1, 2, 3]]  - ravel('F') -> [0, 0, 1, 1, 2, 2, 3, 3]
        #
        # close, but ends duplicated. => trim the ends and reshape to f2elm shape
        #
        # [[0, 1, 2]
        #  [1, 2, 3]]
        #
        f2elm = np.vstack((faces, faces)).ravel(
            order='F')[1:-1].reshape(f2elm.shape, order='F')
        return f2elm

    def edge2elm_ed(self, arr):
        """ internal method to move edge values (defined on the interfaces)
        to values on the "element edge space", the edge dof interior to each element
        @param arr  array formatted on edge space (nFaces,)
        @retval elmEdArr  array formatted on "element edge space" (2, nElm)
        """
        return arr[self.ed2elm]
```

# computation of fluxes

The 'back end' of an explicit DG computation is the unrolled vector of all the problem unknowns. The front end that we'd like to interact with is the dgnodes data structure

```python
def plot_solution(ax, u, dofh):
    """ u formatted like dgnodes """
    for elm in range(dofh.nElm):
        nodal_pts = dofh.dgnodes[:, elm]
        nodal_values = u[:, elm]
        ax.plot(nodal_pts, nodal_values)
    return ax
```

```python
# Low storage Runge-Kutta coefficients LSERK
rk4a = np.array([
         0.0,
        -567301805773.0/1357537059087.0,
        -2404267990393.0/2016746695238.0,
        -3550918686646.0/2091501179385.0,
        -1275806237668.0/842570457699.0])

rk4b = np.array([
         1432997174477.0/9575080441755.0,
         5161836677717.0/13612068292357.0,
         1720146321549.0/2090206949498.0,
         3134564353537.0/4481467310338.0,
         2277821191437.0/14882151754819.0])

rk4c = np.array([
         0.0,
         1432997174477.0/9575080441755.0,
         2526269341429.0/6820363962896.0,
         2006345519317.0/3224310063776.0,
         2802321613138.0/2924317926251.0])
```

```python
# constants
π = np.pi

# geometry set up
P = np.linspace(0, 2*π, 10)
mesh1d = Mesh1D(P)
master = Master1D(p=2)
dofh = DG_dofh_1D(mesh1d, master)
mapdgn = np.zeros((dofh.dgnodes.shape[0], 1, dofh.dgnodes.shape[1]))
mapdgn[:,0,:] = dofh.dgnodes
_map = fem_map.Affine_Mapping(master=[master], dgnodes=[mapdgn])
```

We choose numerical fluxes of the form

$$\widehat{au} =\left\{\!\!\left\{au\right\}\!\!\right\} + (1-\alpha)\frac{|a|}{2}  \left[\!\!\left[u\right]\!\!\right]$$

Where $\alpha = 0$ represents an upwinded flux and $\alpha=1$ represents a central flux. These are shown in Hesthaven to be stable for the equation we are interested in solving.

```python
def compute_interior_flux(u, norm, dofh, α):
    """ computes the numerical flux at all of the element interfaces
    @param u  the current solution u, unrolled to a vector
    NOTE: boundary interfaces will be filled with garbage, and must be corrected
    """
    pm = dofh.lg_PM

    # equivalent to the flux
    #  \hat{au} = {{au}} + (1-α) * |a|/2 * [[u]]
    # at element interfaces. First and last interface will have garbage.
    flux = a/2*(u[pm['-']] + u[pm['+']]) + (1-α)*np.abs(norm*a)/2.*(u[pm['+']] - u[pm['-']])
    return flux
```

# semi-discrete scheme

Considering the "weak" DG-FEM form,  we have the semi-discrete element local equation

\begin{align}
    \int_K \frac{\partial u_h}{\partial t} v \, dK
    +\int_{K} (au_h) \frac{\partial v}{\partial x} \, dK =
    -\int_{\partial K} \hat{n}\cdot \widehat{au} v \, d\partial K
\end{align}

Choosing a representation $u=u_i\phi_i$ piecewise polynomial over each element, and the same test space, we have, for a given choice of numerical flux $\widehat{au}$, and noting that in 1D, the normal vectors are simply (-1, +1):

\begin{align}
    \int_K \frac{\partial}{\partial t} (u_i(t) \phi_i(x)) \phi_j(x) \, dx
    +\int_{K} a(u_i(t)\phi_i(x)) \frac{\partial \phi_j(x)}{\partial x} \, dx =
    -(\widehat{au}(x_R) - \widehat{au}(x_L))
\end{align}

transforming the integrals to the reference element:

\begin{align}
    \int_{\hat{K}} \frac{\partial}{\partial t} (u_i(t) \phi_i(\xi)) \phi_j(\xi) \,|det(J)|\, d\xi
    +\int_{\hat{K}} a(u_i(t)\phi_i(\xi)) \frac{\partial \phi_j(\xi)}{\partial \xi} \, |det(J)|\, d\xi =
    -(\widehat{au}(x_R) - \widehat{au}(x_L))
\end{align}

This completes the description of the semi-discrete scheme, and we have a choice as to how to compute these integrals. The important part is that since the coefficients $u_i$ vary in time but are constants with respect to space, we can write

\begin{align}
    &\frac{\partial u_i(t)}{\partial t} \int_{\hat{K}} \phi_i(\xi) \phi_j(\xi) \,|det(J)|\, d\xi
    +au_i\int_{\hat{K}} \phi_i(\xi) \left(\frac{d\xi}{dx}\right)\frac{\partial \phi_j(\xi)}{\partial \xi} \, |det(J)|\, d\xi =
    -(\widehat{au}(x_R) - \widehat{au}(x_L)) \\
   &\Rightarrow M_K \vec{\frac{du_h}{dt}} + a  S_K \vec{u_h}
   = - L\, (\widehat{au}(x_R) - \widehat{au}(x_L))
\end{align}

Where we have computed $M_K$ and $S_K$, the mass and stiffness matrices for element $K$. Although we would normally do this with a quadrature rule, we can take advantage of the fact that in 1D (and indeed under any affine mapping from reference to physical element), $J^{-1}$ and $|\det(J)|$ will be constant over the entire element (also note that in 1D, $J^{-1}$ is a 1x1 matrix)<sup>1</sup>. In that case, we can treat both as constants, precompute $M_{\hat{K}}, S_{\hat{K}}$, and multiply the entire element-local equation by $M^{-1}$, giving

\begin{align}
\vec{\frac{du_h}{dt}} &= - a \frac{\det(J)_K}{\det(J)_K}\, J^{-1}_K M_{\hat{K}}^{-1}S^T_{\hat{K}} \vec{u_h}
  - \frac{1}{\det(J)_K} M^{-1}_K L\, (\widehat{au}(x_R) - \widehat{au}(x_L)) \\
  &= - a \, J^{-1}_K M_{\hat{K}}^{-1}S^T_{\hat{K}} \vec{u_h}
  - \frac{1}{\det(J)_K} M^{-1}_K L\, (\widehat{au}(x_R) - \widehat{au}(x_L))
\end{align}

Which is a good form for a black box integrator, since we have a "naked" $\frac{du_h}{dt}$, and because the scheme is explicit.

note<sup>1</sup>: $J, J^{-1}$ are 1x1 matrices, and $\det{J}$ is simply $J_{11}$; $J^{-1} = 1/J_{11}$. It's important for the clarity of explicit schemes to understand where these cancellations occur.

```python
def advect_rhs_1D(u, t_local, a, dofh, _map, master, flux_fn, gD, norm):

    return u
```

```python
# final time
T = 10

# compute time step size, irrelevant for backward euler
CFL = 0.75
Δx = dofh.dgnodes[1,0] - dofh.dgnodes[0,0]
Δt = CFL/(2*π)*Δx
Δt = Δt / 2

# number of timesteps needed
steps = int(np.ceil(T/Δt))

# initial condition, advection speed
solution = np.zeros((steps, *dofh.dgnodes.shape))
a = 2 * np.pi
solution[0,::] = np.sin(dofh.dgnodes)

LSERK_stages = [0, 1, 2, 3, 4]
t = 0

gD = lambda t: -np.sin(a*t)

# normal vectors, all positive
norm = np.ones((2, dofh.nElm))
norm[0,:] *= -1

# function pointer to something that can compute fluxes
flux_fn = compute_interior_flux

# time loop
RK_resid = np.zeros_like(dofh.dgnodes)
for tstep in range(3):
    u = solution[tstep, ::]
    for s in LSERK_stages:
        t_local = t + rk4c[s]*Δt
        rhsu = advect_rhs_1D(u, t_local, a, dofh, _map, master, flux_fn, gD, norm)
        RK_resid = rk4a[s]*RK_resid + Δt*rhsu
        u += rk4b[s]*RK_resid
    t += Δt
```

```python
u0 = solution[0,:,:]
fix, ax = plt.subplots()
ax = plot_solution(ax, u0, dofh)
```

```python
pm = dofh.lg_PM
u = u0.ravel()

# normal vectors on interfaces, all positive
norm_faces = np.ones(pm['-'].shape[0])

α = 0

# compute interior fluxes
flux = compute_interior_flux(u, norm_faces, dofh, α)

# compute boundary fluxes
flux[0] = gD(t_local)
flux[-1] = flux[0]

dofh.edge2elm_ed(flux)
```

    array([[-0.37763258,  5.55594966,  6.31799283, -5.55594966, -2.19421588,
             2.19421588, -4.12377695,  4.12377695,  6.31799283],
           [ 5.55594966,  6.31799283, -5.55594966, -2.19421588,  2.19421588,
            -4.12377695,  4.12377695,  6.31799283, -6.31799283]])

```python

```


# Algoritmo de Denavit - Hartenberg

## Calculo simbólico

Dentro del notebook de Jupyter tenemos la ventaja de utilizar diferentes tipos de elementos desplegables en un página web, como imagenes, interfaces graficas, etc. En esta práctica utilizaremos un nuevo tipo de elementos que nos ayudan a desplegar matemáticas de una manera estetica y funcional.

Como algunos ya se habran dado cuenta, dentro de un notebook tenemos diferentes tipos de celdas, las que aparecen por default cuando creas una nueva celda son celdas de código, pero si queremos una celda de comentarios, tan solo tenemos que cambiarla en el menu de arriba por el tipo `Markdown`.

En este tipo de celda podemos incluso añadir imagenes como la que esta arriba (si quieres ver como se hace, tan solo tienes que hacer doble clic sobre la celda para desplegar el código utilizado).

En especifico nos interesa poder escribir ecuaciones matemáticas como $x_1 = 10$, o bien

$$ E = m c^2 $$

> Nota que dependiendo del numero de `$`, la ecuación aparecerá en linea junto con el texto, o aparte y centrado.

Pero hasta el momento no hemos hecho ningun cálculo con estas ecuaciones, tan solo las hemos escrito, y a pesar de que se ven bien, no podemos manipularlas de ninguna manera.

Para esto nos ayudará la librería `sympy`, la cual esta diseñada especificamente para poder hacer cálculos simbólicos. Para empezar tenemos que importar algunas funciones de la librería.

```python
# Esta libreria tiene las funciones principales que utilizaremos
from sympy import var, Matrix, Function, sin, cos, pi, trigsimp
# Esta libreria contiene una funcion que la va a dar un formato "bonito" a nuestras ecuaciones
from sympy.physics.mechanics import mechanics_printing
mechanics_printing()
τ = 2*pi
```

Una vez importadas las funciones de nuestra librería, podemos empezar declarando variables (o constantes) conocidas para nosotros.

```python
var("t q1 q2 q3")
```

```python
var("l1:4")
```

> Nota que cualquiera de estas dos notaciones es valida para importar variables de sympy

Si ahora creamos una matriz con nuestros datos es posible utilizar las operaciones de matrices conocidas por nosotros, por ejemplo vamos hacer una transformación de movimientos rígidos, definida por la siguiente ecuación:

$$ p_1^0 = R_1 p_0^0 + d_1^0 $$

```python
# Defino el punto origen, la rotación y traslación aplicados
p0 = Matrix([[2],
             [3],
             [0]])
R1 = Matrix([[cos(q1), -sin(q1), 0],
             [sin(q1), cos(q1), 0],
             [0, 0, 1]])
d1 = Matrix([[l1],
             [0],
             [0]])
```

```python
p1 = R1*p0 + d1
p1
```

## Convención Denavit - Hartenberg

Sabemos que la conveción Denavit - Hartenberg es una manera de obtener las transformaciones homogéneas necesarias para cada eslabon de un manipulador, sin embargo las multiplicaciones de matrices resultantes de este algorimto por lo general consumen demasiado tiempo, por lo que en esta práctica aprenderemos a definir una función que calcule cada matriz de transformación y poder multiplicarlas con la computadora.

Lo primero que necesitamos es la matriz general del algoritmo Denavit - Hartenberg:

$$
A_i =
\begin{pmatrix}
    c_{\theta_i} & -s_{\theta_i} c_{\alpha_i} & s_{\theta_i} s_{\alpha_i} & a_i c_{\theta_i} \\
    s_{\theta_i} & c_{\theta_i} c_{\alpha_i} & -c_{\theta_i} s_{\alpha_i} & a_i s_{\theta_i} \\
    0 & s_{\alpha_i} & c_{\alpha_i} & d_i \\
    0 & 0 & 0 & 1
\end{pmatrix}
$$

Si escribimos esta matriz dando valores aleatorios a $\theta$, $\alpha$, $d$ y $a$, tendremos:

```python
a = 1.2
d = 0
α = τ/4
θ = q1

A1 = Matrix([[cos(θ), -sin(θ)*cos(α), sin(θ)*sin(α), a*cos(θ)],
             [sin(θ), cos(θ)*cos(α), -cos(θ)*sin(α), a*sin(θ)],
             [0, sin(α), cos(α), d],
             [0, 0, 0, 1]])
```

```python
A1
```

Dando otros valores aleatorios a la misma matriz tendremos:

```python
a = 0.8
d = 0
α = 0
θ = q2

A2 = Matrix([[cos(θ), -sin(θ)*cos(α), sin(θ)*sin(α), a*cos(θ)],
             [sin(θ), cos(θ)*cos(α), -cos(θ)*sin(α), a*sin(θ)],
             [0, sin(α), cos(α), d],
             [0, 0, 0, 1]])
```

```python
A2
```

Y de la misma manera que en cualquier motor de algebra, si queremos multiplicar, tan solo tenemos que hacerlo:

```python
A1*A2
```

o bien, obtener la matriz inversa:

```python
A1.inv()
```

la cual, a veces puede ser simplificada:

```python
trigsimp(A1.inv())
```


```python
import sys, os
sys.path.insert(0, os.path.join(os.pardir, 'src'))
from approx1D import interpolation, comparison_plot
from Lagrange import Lagrange_polynomials
import sympy as sym

x = sym.Symbol('x')
Omega = [0,1]
N_values = 3, 7, 11, 15

for s in 5, 20:
    f = -sym.tanh(s*(x-0.5))  # sympy expression
    for distribution in 'uniform', 'Chebyshev':
        for N in N_values:
            phi, points = Lagrange_polynomials(
                x, N, Omega,
                point_distribution=distribution)

            u, c = interpolation(f, phi, points)
            filename = 'tmp_tanh_%d_%d_%s' % (N, s, distribution)
            comparison_plot(f, u, Omega, filename,
                            plot_title='s=%g, N=%d, %s points' %
                            (s, N, distribution))
        # Combine plot files (2x2)
        for ext in 'png', 'pdf':
            cmd = 'doconce combine_images ' + ext + ' '
            cmd += ' '.join([
                'tmp_tanh_%d_%d_%s' % (N, s, distribution)
                for N in N_values])
            cmd += ' tanh_Lagrange_%s_s%s' % (distribution, s)
            os.system(cmd)
```

```python

```

Data Science and Machine Learning/Machine-Learning-In-Python-THOROUGH/EXAMPLES/FINITE_ELEMENTS/INTRO/EXERCICES/22_TANH_LAGRANGE.ipynb


```python
import sympy as sym
import numpy as np
```

```python
sym.init_printing()
```

We want the variance of $(\sigma^2_{t+1})^2$ (It equals the variance of $\sigma^2_{t+1}$. By the variance decomposition this equals $\mathbb{E}[\sigma^8_{t+1} | \sigma^2_t]  - \mathbb{E}[\sigma^4_{t+1} | \sigma^2_t]^2$.

The first term is the 4rth-moment of the volatility. We can compute this using the fact that Laplace transforms generate the moments through differentation.

$\mathbb{E}[\sigma^8_{t+1} | \sigma^2_t]
= (-1)^4 \frac{\partial^4}{\partial x^4} \mathbb{E}[\exp(- (-x) \sigma^2_{t+1}) | \sigma^2_t)]
= \frac{\partial^4}{\partial x^4} \exp\left(-A(-x) \sigma^2_t - B(-x)\right) |_{x=0}
= -\frac{\partial^4}{\partial x^4} A(-x) \sigma^2_t - \frac{\partial x^4}{\partial x^4} B(-x) |_{x=0}$

```python
sym.__version__
```

```python
x, y = sym.symbols('x y')
rho, c, delta = sym.symbols('rho c delta')
sigma2 = sym.symbols('sigma^2_t')
```

```python
A =  (rho * x) / (1 + c * x)
B = delta * sym.log(1 + c * x)
```

```python
fourth_moment = (- A.replace(x, -x).diff(x, 4).replace(x,0) * sigma2
                 - B.replace(x,-x).diff(x,4).replace(x,0)).simplify()
```

```python
fourth_moment
```

Similarly, the 2nd-moment of the volatility as a similar expression, we just replace the 4 with a 2.

```python
second_moment = (- A.replace(x, -x).diff(x, 2).replace(x,0) * sigma2
                 - B.replace(x,-x).diff(x,2).replace(x,0)).simplify()
```

```python
variance_variance = sym.factor(fourth_moment - second_moment**2)
```

```python
variance_variance
```

```python
variance_variance.factor().replace(sigma2,1).simplify()
```

```python

```


# Notebook 03: Inverse design parameterization

This notebook will introduce a few basic parameterization concepts for inverse design. The same mode converter device concept as in the previous notebook will be used here as an example.

*Parameterization* refers to how we are representing our device. In the previous notebook, we simply used a 2D array of permittivity values, $\epsilon_r[x,y]$.
However, we observed that this led to continuously varying features, which is not ideal for fabricating devices generated by our optimization.
In this notebook, we will use a modified parameterization of the device to encourage more *binarized* features in the optimized device.
However, as we will see, we will still need to tune various hyperparameters in order to get desirable results.

We begin by importing the necessary python packages:

```python
import numpy as np
import autograd.numpy as npa

import copy

import matplotlib as mpl
mpl.rcParams['figure.dpi']=100

import matplotlib.pylab as plt

from autograd.scipy.signal import convolve as conv
from skimage.draw import circle

import ceviche
from ceviche import fdfd_ez, jacobian
from ceviche.optimizers import adam_optimize
from ceviche.modes import insert_mode

import collections
# Create a container for our slice coords to be used for sources and probes
Slice = collections.namedtuple('Slice', 'x y')
```

## Introduction: device parameterization

Here, our device will be parameterized by a 2D array of density values (a density distribution, referred to as $\rho[x,y]$).
This part of the parameterization actually looks very similar to the 2D array of permitivitty values we used in the previous notebook.
However, the difference is that we then definite two relatively simple operators that will be applied to this density. They are:

 - **Blur operator**: A 2D convolution blur filter with a configurable radius; can be applied a configurable number of times
 - **Projection operator**: A sigmoid-like nonlinear function for *binarizing* the output materials; has tunable slope and transition region

After these two operations are applied, the resulting density distribution, $\tilde{\rho}[x,y]$, will be a 2D array with values varying between `0.0` and `1.0`. A value of `0.0` represents a *background* material, while a value of `1.0` represents a *foreground* material.
The permitivitty distribution, which represents the optical description of our device, is then constructed as:
\begin{equation}
    \epsilon_r[x,y] = \epsilon_{\text{min}} + \left(\epsilon_{\text{max}} - \epsilon_{\text{min}} \right) \tilde{\rho}{[x,y]}
\end{equation}

We start by defining our parameterization operators:

```python
# Projection that drives rho towards a "binarized" design with values either 0 or 1
def operator_proj(rho, eta=0.5, beta=100, N=1):
    """Density projection
    eta     : Center of the projection between 0 and 1
    beta    : Strength of the projection
    N       : Number of times to apply the projection
    """
    for i in range(N):
        rho =  npa.divide(npa.tanh(beta * eta) + npa.tanh(beta * (rho - eta)),
                          npa.tanh(beta * eta) + npa.tanh(beta * (1 - eta)))

    return rho

# Blurring filter that results in smooth features of the structure
# First we define a function to create the kernel
def _create_blur_kernel(radius):
    """Helper function used below for creating the conv kernel"""
    rr, cc = circle(radius, radius, radius+1)
    kernel = np.zeros((2*radius+1, 2*radius+1), dtype=np.float)
    kernel[rr, cc] = 1
    return  kernel/kernel.sum()

# Then we define the function to apply the operation
def operator_blur(rho, radius=2, N=1):
    """Blur operator implemented via two-dimensional convolution
    radius    : Radius of the circle for the conv kernel filter
    N         : Number of times to apply the filter

    Note that depending on the radius, the kernel is not always a
    perfect circle due to "pixelation" / stair casing
    """

    kernel = _create_blur_kernel(radius)

    for i in range(N):
        # For whatever reason HIPS autograd doesn't support 'same' mode, so we need to manually crop the output
        rho = conv(rho, kernel, mode='full')[radius:-radius,radius:-radius]

    return rho
```

### Visualization of the blur

Below we visualize the blur operator applied to a random 2D array:

```python
# Specify a range of values for the blur radius
blur_radii = [2, 3, 4, 5, 6]

# Number of times to apply the blur filter
N_blur = 1

# Define a random 2D array to test our blur filter on
rho = np.random.rand(50, 50)

# Create a figure with panels to plot into
fig, axs = plt.subplots(2, len(blur_radii)+1,figsize=(12,4), constrained_layout=True)

# First, plot a dummy conv filter
axs[0,0].set_title('Initial random image')
axs[0,0].imshow(np.zeros((1,1)), vmin=0, cmap='Greys')
# And also plot the initial random image
axs[1,0].imshow(rho)

# Now, loop over the blur radii and visualize each result
for i, radius in enumerate(blur_radii):
    kernel = _create_blur_kernel(radius)

    kernel_pad = np.pad(kernel, (2+np.max(blur_radii)-radius, 2+np.max(blur_radii)-radius),
                        'constant', constant_values=(0, 0))
    axs[0,i+1].imshow(kernel_pad, vmin=0)

    axs[0,i+1].set_title('radius = %d' % radius)

    rho_p = operator_blur(rho, radius=radius, N=N_blur)
    axs[1,i+1].imshow(rho_p)

axs[0,0].set_yticks([])
axs[0,0].set_xticks([])
axs[0,0].set_ylabel('Conv filter')
axs[1,0].set_ylabel('Image')

fig.align_labels()
plt.show()
```

### Visualization of the projection

Below we visualize the projection operator:

```python
rho = np.linspace(-0.5, +1.5, 999)

# Visualize different values of the projection strength
plt.figure()
for beta in [5, 10, 50, 100]:
    plt.plot(rho, operator_proj(rho, beta=beta), label=r"$\beta$ = %d" % beta)

plt.xlabel(r"Input $\rho$")
plt.ylabel(r"Projected $\hat{\rho}$")
plt.legend()
plt.show()
```

### Visualization of blur + projection

Finally, we visualize the combined projection and blur operation. Notice how the larger blur radius leads to larger and smoother features.

```python
# Specify a range of values for the blur radius
blur_radii = [2, 3, 4, 5, 6, 10]

# Number of times to apply the blur filter
N_blur = 1

# Number of times to apply the projection operator
N_proj = 1

# Specify beta value to use for projection
beta = 200

# Specify eta value to use for projection
eta = 0.5

# Define a random 2D array to test our blur filter on
rho = np.random.rand(100, 100)

# Create a figure with panels to plot into
fig, axs = plt.subplots(2, len(blur_radii)+1,figsize=(12,4), constrained_layout=True)

# First, plot a dummy conv filter
axs[0,0].set_title('Initial random image')
axs[0,0].imshow(np.zeros((1,1)), vmin=0, cmap='Greys')
# And also plot the initial random image
axs[1,0].imshow(rho)

# Now, loop over the blur radii and visualize each result
for i, radius in enumerate(blur_radii):
    kernel = _create_blur_kernel(radius)

    kernel_pad = np.pad(kernel, (2+np.max(blur_radii)-radius, 2+np.max(blur_radii)-radius),
                        'constant', constant_values=(0, 0))
    axs[0,i+1].imshow(kernel_pad, vmin=0)

    axs[0,i+1].set_title('radius = %d' % radius)

    rho_p = operator_blur(rho, radius=radius, N=N_blur)
    rho_p = operator_proj(rho_p, beta=beta, eta=eta, N=N_proj)
    axs[1,i+1].imshow(rho_p)

axs[0,0].set_yticks([])
axs[0,0].set_xticks([])
axs[0,0].set_ylabel('Conv filter')
axs[1,0].set_ylabel('Image')

fig.align_labels()
plt.show()
```

---

## Simulation and optimization parameters

```python
# Angular frequency of the source in 1/s
omega=2*np.pi*200e12
# Spatial resolution in meters
dl=40e-9
# Number of pixels in x-direction
Nx=120
# Number of pixels in y-direction
Ny=120
# Number of pixels in the PMLs in each direction
Npml=20
# Minimum value of the relative permittivity
epsr_min=1.0
# Maximum value of the relative permittivity
epsr_max=12.0
# Radius of the smoothening features
blur_radius=5
# Number of times to apply the blur
N_blur=1
# Strength of the binarizing projection
beta=500.0
# Middle point of the binarizing projection
eta=0.5
# Number of times to apply the blur
N_proj=1
# Space between the PMLs and the design region (in pixels)
space=10
# Width of the waveguide (in pixels)
wg_width=12
# Length in pixels of the source/probe slices on each side of the center point
space_slice=8

# Number of epochs in the optimization
Nsteps=50
# Step size for the Adam optimizer
step_size=5e-3
```

## Utility functions

As in **Notebook 02**, here we define several utility functions for constructing the simulation domain, feed waveguides, design region, input source, and output probe,

```python
def init_domain(Nx, Ny, Npml, space=10, wg_width=10, space_slice=5):
    """Initializes the domain and design region

    space       : The space between the PML and the structure
    wg_width    : The feed and probe waveguide width
    space_slice : The added space for the probe and source slices
    """
    rho = np.zeros((Nx, Ny))
    bg_rho = np.zeros((Nx, Ny))
    design_region = np.zeros((Nx, Ny))

    # Input waveguide
    bg_rho[0:int(Npml+space),int(Ny/2-wg_width/2):int(Ny/2+wg_width/2)] = 1

    # Input probe slice
    input_slice = Slice(x=np.array(Npml+1),
        y=np.arange(int(Ny/2-wg_width/2-space_slice), int(Ny/2+wg_width/2+space_slice)))

    # Output waveguide
    bg_rho[int(Nx-Npml-space)::,int(Ny/2-wg_width/2):int(Ny/2+wg_width/2)] = 1

    # Output probe slice
    output_slice = Slice(x=np.array(Nx-Npml-1),
                         y=np.arange(int(Ny/2-wg_width/2-space_slice), int(Ny/2+wg_width/2+space_slice)))

    design_region[Npml+space:Nx-Npml-space, Npml+space:Ny-Npml-space] = 1

    # Const init
    rho[Npml+space:Nx-Npml-space, Npml+space:Ny-Npml-space] = 0.5
    # Random init
    # rho = design_region * np.random.rand(Nx, Ny)

    return rho, bg_rho, design_region, input_slice, output_slice

def viz_sim(epsr, source, slices=[]):
    """Solve and visualize a simulation with permittivity 'epsr'
    """
    simulation = fdfd_ez(omega, dl, epsr, [Npml, Npml])
    Hx, Hy, Ez = simulation.solve(source)

    fig, ax = plt.subplots(1, 2, constrained_layout=True, figsize=(6,3))
    ceviche.viz.real(Ez, outline=epsr, ax=ax[0], cbar=False)
    for sl in slices:
        ax[0].plot(sl.x*np.ones(len(sl.y)), sl.y, 'b-')
    ceviche.viz.abs(epsr, ax=ax[1], cmap='Greys');
    plt.show()

    return (simulation, ax)

def mask_combine_rho(rho, bg_rho, design_region):
    """Utility function for combining the design region rho and the background rho
    """
    return rho*design_region + bg_rho*(design_region==0).astype(np.float)

def epsr_parameterization(rho, bg_rho, design_region, radius=2, N_blur=1, beta=100, eta=0.5, N_proj=1):
    """Defines the parameterization steps for constructing rho
    """
    # Combine rho and bg_rho; Note: this is so the subsequent blur sees the waveguides
    rho = mask_combine_rho(rho, bg_rho, design_region)

    rho = operator_blur(rho, radius=radius, N=N_blur)
    rho = operator_proj(rho, beta=beta, eta=eta, N=N_proj)

    # Final masking undoes the blurring of the waveguides
    rho = mask_combine_rho(rho, bg_rho, design_region)

    return epsr_min + (epsr_max-epsr_min) * rho

def mode_overlap(E1, E2):
    """Defines an overlap integral between the sim field and desired field
    """
    return npa.abs(npa.sum(npa.conj(E1)*E2))
```

## Simulate the initial structure

```python
# Initialize the parametrization rho and the design region
rho, bg_rho, design_region, input_slice, output_slice = \
    init_domain(Nx, Ny, Npml, space=space, wg_width=wg_width, space_slice=space_slice)

# Compute the permittivity from rho_init, including blurring and projection
epsr_init = epsr_parameterization(rho, bg_rho, design_region, \
                                  radius=blur_radius, N_blur=N_blur, beta=beta, eta=eta, N_proj=N_proj)

# Setup source
source = insert_mode(omega, dl, input_slice.x, input_slice.y, epsr_init, m=1)

# Setup probe
probe = insert_mode(omega, dl, output_slice.x, output_slice.y, epsr_init, m=2)

# Simulate initial device
simulation, ax = viz_sim(epsr_init, source, slices=[input_slice, output_slice])

# get normalization factor (field overlap before optimizing)
_, _, Ez = simulation.solve(source)
E0 = mode_overlap(Ez, probe)
```

## Run the mode converter optimization

```python
def objective(rho):
    """Objective function called by optimizer

    1) Takes the density distribution as input
    2) Constructs epsr
    2) Runs the simulation
    3) Returns the overlap integral between the output wg field
       and the desired mode field
    """
    rho = rho.reshape((Nx, Ny))
    epsr = epsr_parameterization(rho, bg_rho, design_region, \
                             radius=blur_radius, N_blur=N_blur, beta=beta, eta=eta, N_proj=N_proj)
    simulation.eps_r = epsr
    _, _, Ez = simulation.solve(source)
    return mode_overlap(Ez, probe) / E0

# Compute the gradient of the objective function using revere-mode differentiation
objective_jac = jacobian(objective, mode='reverse')

# Maximize the objective function using an ADAM optimizer
(rho_optimum, loss) = adam_optimize(objective, rho.flatten(), objective_jac,
                         Nsteps=Nsteps, direction='max', step_size=step_size)

# Simulate optimal device
rho_optimum = rho_optimum.reshape((Nx, Ny))
epsr = epsr_parameterization(rho_optimum, bg_rho, design_region, \
                                  radius=blur_radius, N_blur=N_blur, beta=beta, eta=eta, N_proj=N_proj)

viz_sim(epsr, source, slices=[input_slice, output_slice]);
```

## Visualizing the parameterization steps

We can take a look what each step of our parameterization actually looks like. Below we visualize the raw density distribution, the blurred density distribution, and finally, the projected device density.

```python
fig, axs = plt.subplots(1,3,constrained_layout=True,sharex=True,sharey=True, figsize=(8.5,3))
Z = mask_combine_rho(rho_optimum, bg_rho, design_region)
ceviche.viz.abs(Z, cmap='Greys', ax=axs[0])
axs[0].set_xlabel('')
axs[0].set_xticks([])
axs[0].set_ylabel('')
axs[0].set_yticks([])
axs[0].set_title('Raw input density')

Z = mask_combine_rho(rho_optimum, bg_rho, design_region)
Z = operator_blur(Z, radius=blur_radius, N=N_blur)
# Z = operator_proj(Z, beta=beta, eta=eta)
Z = mask_combine_rho(Z, bg_rho, design_region)
ceviche.viz.abs(Z, cmap='Greys', ax=axs[1])
axs[1].set_xlabel('')
axs[1].set_ylabel('')
axs[1].set_title('Blurred density')

Z = rho_optimum
Z = mask_combine_rho(Z, bg_rho, design_region)
Z = operator_blur(Z, radius=blur_radius, N=N_blur)
Z = operator_proj(Z, beta=beta, eta=eta, N=N_proj)
Z = mask_combine_rho(Z, bg_rho, design_region)
ceviche.viz.abs(Z, cmap='Greys', ax=axs[2])
axs[2].set_xlabel('')
axs[2].set_ylabel('')
axs[2].set_title('Projected density (final structure)');
```

## Penalizing the amount of material

We notice that in the optimized device shown above that there seems to be a lot of unnecessary material in the device. Here, we will try to eliminate some of this extra material by adding a penalty term to the objective function.

To add the penalty term, we only need to modify the objective function:

```python
def objective(rho, penalty_weight=1e8):
    rho = rho.reshape((Nx, Ny))
    epsr = epsr_parameterization(rho, bg_rho, design_region, \
                             radius=blur_radius, N_blur=N_blur, beta=beta, eta=eta, N_proj=N_proj)
    simulation.eps_r = epsr
    _, _, Ez = simulation.solve(source)

    # This penalty term is directly proportional to the material area
    # penalty = penalty_weight * (design_region*(epsr-1)).sum() # penalty_weight = 1e-10

    # This penalty term is the L2-norm of the raw density
    penalty = penalty_weight * npa.linalg.norm(rho)

    return mode_overlap(Ez, probe) / E0 - penalty
```

```python
# Initialize the parametrization rho and the design region
rho, bg_rho, design_region, input_slice, output_slice = \
    init_domain(Nx, Ny, Npml, space=space, wg_width=wg_width, space_slice=space_slice)

# Compute the permittivity from rho_init, including blurring and projection
epsr_init = epsr_parameterization(rho, bg_rho, design_region, \
                                  radius=blur_radius, N_blur=N_blur, beta=beta, eta=eta, N_proj=N_proj)

# Setup source
source = insert_mode(omega, dl, input_slice.x, input_slice.y, epsr_init, m=1)

# Setup probe
probe = insert_mode(omega, dl, output_slice.x, output_slice.y, epsr_init, m=2)

# Simulate initial device
viz_sim(epsr_init, source, slices=[input_slice, output_slice])

# Run optimization
objective_jac = jacobian(objective, mode='reverse')
(rho_optimum, loss) = adam_optimize(objective, rho.flatten(), objective_jac,
                         Nsteps=Nsteps, direction='max', step_size=step_size)

# Simulate optimal device
rho_optimum = rho_optimum.reshape((Nx, Ny))
epsr_pen = epsr_parameterization(rho_optimum, bg_rho, design_region, \
                                  radius=blur_radius, N_blur=N_blur, beta=beta, eta=eta, N_proj=N_proj)
viz_sim(epsr_pen, source, slices=[input_slice, output_slice]);
```

We can then quantify how much our penalization impacted the amount of material used in the final design.

```python
# Calculate areas
def calc_design_area(design_region, epsr_max, epsr_min, epsr, dl):
    """Computes the area of material used in the design region
    """
    A = ((epsr-epsr_min)/epsr_max * design_region).sum() * dl**2 * 1e12
    A_design_region = design_region.sum() * (dl)**2 * 1e12
    return A, A_design_region

_, A_design_region = calc_design_area(design_region, epsr_max, epsr_min, epsr, dl)
A_original, _ = calc_design_area(design_region, epsr_max, epsr_min, epsr, dl)
A_pen, _ = calc_design_area(design_region, epsr_max, epsr_min, epsr_pen, dl)

# Print summary
print('Design region area: %.2f um^2' % A_design_region)
print('Unpenalized design area: %.2f um^2' % A_original)
print('Penalized design area: %.2f um^2' % A_pen)
print('---')
print('Improvement: %.2f%%' % (100*(1-A_pen/A_original)))
```

    Design region area: 5.76 um^2
    Unpenalized design area: 1.96 um^2
    Penalized design area: 1.03 um^2
    ---
    Improvement: 47.32%

```python

```

```python

```


## Example: Point Source (hw 12)

Previously, we calculated the geometric attenuation of radiation from a point source, to identify the flux, $\phi(r)$ as a function of $r$, distance from the source of neutrons.

The uncollided flux for a point source (strength $S_p$) in an infinite medium with attenuation coefficient $\mu$ would be:

\begin{align}
&\phi^o(r) = \frac{S_p}{4\pi r^2}e^{-\mu r}\\
\mbox{where}&\\
&e^{-\mu r}=\mbox{material attenuation}
\end{align}

If the medium is not merely absorptive, but is a scattering medium, we can use the notion of diffusion to define the flux at the point r in an infinite moderating (scattering) medium.

\begin{align}
\phi(r) &= \frac{S_p}{4\pi Dr}e^{-\mu r}\\
\mbox{where }&\\
D &= \mbox{ the diffusion coefficient in the medium}
\end{align}

To show that the above is true, one must solve solution to the diffusion equation. To do so, one must use spherical coordinates for the laplacian and two boundary conditions. First, the finite flux condition, such that the flux is zero at infinity. Second, the source condition, defining the limit of the neutron current density as r goes to zero.

####  Recall: What are the units of D?

To remind ourselves of current density, let's identify the neutron current density in this problem. Specifically, let's ask: **What is the net number of neutrons flowing out through a spherical shell of radius r surrounding the source?**

This will be the neutron current density normal to the surface of a sphere at r. First, the neutron current density follows Fick's law:

\begin{align}
\boldsymbol{\vec{J}} = -D\nabla \phi
\end{align}

And in our case, the problem has radial symmetry, so we're interested in $\vec{J}(r)$. We'll need to use the r-component of the gradient operator in spherical coordinates ($\nabla_r$).

\begin{align}
\boldsymbol{\vec{J}}_r &= -D\nabla_r \phi(r)\\
&= -D \frac{d\phi(r)}{dr}\hat{r}
\end{align}

This gives:

\begin{align}
\boldsymbol{\vec{J}}_r &= -D \frac{d\phi(r)}{dr}\hat{r}\\
&= -D \frac{d}{dr}\left(\frac{S_p}{4\pi Dr}e^{-\mu r}\right)\hat{r}\\
&= \frac{S_p}{4\pi}\left(\frac{1}{r^2} + \frac{\mu}{r}\right)e^{-\mu r}\hat{r}\\
\end{align}

####  Fundamentals Review: What is $\hat{r}$?

```python

```


>>> Work in Progress

#### Outline
- Perceptron
- Exponential Family
- Generalized Linear Models(GLM)
- Softmax Regression(Multiclass classification)

### Logistic Regression (Recap)
- Logistic Regression uses sigmoid function
- ranges from $-\infty$ to $\infty$, with values ranging from 0 to 1, which is probability
  > $$ g(z) = \frac{1}{1+e^{-z}} $$
  - At z=0, g(z) = 0.5
  - As z tends to $-\infty$, g converges to 0
  - As z tends to $\infty$, g converges to 1

- variant of this is perceptron

### Perceptron

- is not used widely because it does not have a probabilistic interpretation
- is taught for historical reasons
- logistic regression is a softer version of perceptron

> $$\begin{equation}
  g(z) =
    \begin{cases}
      1 & \text{if z $\ge$ 0}\\
      0 & \text{if z $\lt$ 0}
    \end{cases}
\end{equation}$$

- hypothesis function
> $h_{\theta}(x) = g(\theta^{T}x)$

$\tiny{\text{YouTube-Stanford-CS229-Andrew Ng/Anand Avati}}$

> $\theta_{j} := \theta_{j} + \alpha(y^{(i)} - h_{\theta}(x^{(i)}))x_{j}^{(i)}$
- In this equation $(y^{(i)} - h_{\theta}(x^{(i)}))$ is scalar, because $y^{(i)}$ is either 0/1 and so will be $h_{\theta}(x^{(i)})$
- So the result can be either
$\begin{equation}
      =
    \begin{cases}
      0  & \text{if algorithm got it right}\\
      +1 & \text{if wrong $y^{(i)} = 1$}\\
      -1 & \text{if wrong $y^{(i)} = 0$}
    \end{cases}
\end{equation}$
  - A result of 0 means, if the example is already classified, you do nothing
  - A result of +1/-1 means, the example is misclassified, and you either add/subtract a small component of the example ($\alpha x_{j}^{(i)}$)
  - This will shift the decision boundary correctly

$\tiny{\text{YouTube-Stanford-CS229-Andrew Ng/Anand Avati}}$

### Exponential Family
- its a class of probability distribution
- they are closely related to GLM

- PDF:
  > $P(y;\eta) = b(y) \text{exp}[\eta^{T}T(y) - a(\eta)]$
    - y: data - use it to model output of data
    - $\eta$: natural parameter
    - $T(y)$: sufficient statistics, cannot involve $\eta$
    - b(y): base measure, cannot involve $\eta$
    - $a(\eta)$: log partition function
  > $P(y;\eta) = \frac{b(y) e^{(\eta^{T}T(y))}}{e^{a(\eta)}}$
    - $a(\eta)$: is also called normalizing constant of probability distribution

#### Types of exponential family
##### Bernoulli distribution
- Bernoulli distribution belongs to the exponential family
  - PDF
  >$P(y; \theta) $
  >$= \phi^{y}(1-\phi)^{1-y}$
  >$ = exp(log(\phi^{y}(1-\phi)^{1-y}))$
  >$ = exp[log(\frac{\phi}{1-\phi})y + log(1-\phi)]$
  - where
    >$b(y) = 1$
    >$T(y) = y$
    >$\eta = log(\frac{\phi}{1-\phi}) \Rightarrow \phi = \frac{1}{1+e^{-\eta}}$
    >$a(\eta) = -log(1-\phi) = -log(1-\frac{1}{1+e^{-\eta} }) = log(1+e^{\eta})$
    - We are linking the canonical parameters to natural parameters here

##### Gaussian distribution
- Gaussian distribution - with fixed variance
  - Assume $\sigma^{2} = 1$
  - PDF
  >$P(y; \mu)$
  >$= \frac{1}{\sqrt(2\pi)}exp(-\frac{(y-\mu)^{2}}{2})$
  >$ = \frac{1}{\sqrt(2\pi)} e^{-\frac{y^{2}}{2}}exp(\mu y - \frac{1}{2}\mu ^{2}) $
  - where
    >$b(y) = \frac{1}{\sqrt(2\pi)} e^{-\frac{y^{2}}{2}}$
    >$T(y) = y$
    >$\eta = \mu$
        >$a(\eta) = \frac{\mu^{2}}{2} = \frac{\eta^{2}}{2}$

##### Other distributions:
- How do you decide which distribution to use?
  - The task in reality tells/influences you which distribution to use
  - Real: Gaussian
    - regression - predict house prices
  - Binary: Bernoulli
    - classification
  - Integer Count: Poisson
    - number of visitors to a web page
  - Real positive: Gamma, Exponential
  - Prob distribution over other distribution: Beta, Dirichlet (used mostly in Bayesian statistics)

#### Properties of Exponential family:
* If you perform maximum likelihood MLE wrt $\eta \Rightarrow$ concave
  * NLL is convex (negative log likelihood)
* $E[y;\eta] = \frac{\partial}{\partial \eta} a(\eta)$
* $Var[y;\eta] = \frac{\partial^{2}}{\partial \eta^{2}} a(\eta)$
  * Generally to calculate distribution properties (mean and variance), you need to integrate, in exponential family you differentiate

### Generalized Linear Models (GLM)

Natural extension of exponential family - to include covariance/input features. Powerful models can be made by using this.

Assumption/Design choices (to move from exponential family to GLM):
i) $y | x; \theta \sim $ Exponential family$(\eta) $
ii) $\eta = \theta^{T}x $ where $ \theta \in \mathbb R^{n}, x \in \mathbb R^{n}$
iii) At test time: Output E[y|x; $\theta$]
  - $h_{\theta}(x) = E[y|x; \theta]$ - is the hypothesis function
    - if we plugin exponential family as Gaussian, the hypothesis will turn out to be Gaussian hypothesis of linear regression
    - if we plugin exponential family as Bernoulli, the hypothesis will turn out to be Bernoulli hypothesis of logistic regression

- One way to visualize this is (as in figure below)
  - there is a model (linear model here)
    - given x, there is a learnable parameter $\theta$, and $\theta^{T}x$ will give you a parameter $\eta$
  - there is a distribution
    - the distribution is a member of exponential family and parameter for this distribution is output of linear model
    - we choose exponential family based on the data that we have (classification problem, regression problem, or other)
    - we will choose appropriate b, a and T based on distribution of your choice
  - expectation
    - During test time
      - $E[y; \eta] = E[y; \theta^{T}x] = h_{\theta}(x)$ - this is the hypothesis function
      - Caveat:
        - the parameter that we are learning during gradient descent is $\theta$
        - we dont learn anything of the exponential family eg., $\mu, \sigma^{2}, \eta$
        - we learn $\theta$, that is part of model and not part of distribution
    - During train time
      - we perform gradient ascent/descent on the log probability with y where natural parameter was reparameterized with the linear model
      - the gradient ascent is done by taking gradients on $\theta$

- Question
  - Are we training $\theta$ to predict the parameter of exponential family distribution whose mean is our prediction for y
    - True

$\tiny{\text{YouTube-Stanford-CS229-Andrew Ng/Anand Avati}}$

- This is the reason, or how GLMs are an extension of exponential families. You reparameterize the parameters with the linear model and you get a GLM.

#### GLM training
- At train time, we perform maximum likelihood over the log probability of y with respect to $\theta$
- Learning update rule
  - plugin appropriate $h_\theta(x)$ depending on the choice of distribution and you can start learning
  - the __learning update rule__ is the __same__ for all GLMs, for classification or regression, just the $h_{\theta}$ varies.

> $ \theta _{j} := \theta _{j} - \alpha (h_{\theta}(x^{(i)}) - y^{(i)}).x_{j}^{(i)} $

#### Terminology
- $\eta$ - natural parameter
- $\mu = E[y_{i}; \eta] = g(\eta)$ - natural parameter to the mean of the function - canonical response function
- $\eta = g^{-1}(\mu)$ - canonical link function
- $g(\eta) = \frac{\partial}{\partial \eta}a(\eta)$

- There are 3 types of parameterization being used here:
i) Model parameter - $\theta$ - this is the only parameter that is learned
ii) Natural parameter - $\eta$
iii) Canonical parameter
  - $\phi$ - Bernoulli
  - $\mu, \sigma^{2}$ - Gaussian
  - $\lambda$ - Poisson

- How are they linked
  - Model parameter and Natural parameter are linked by design choice ($\theta^{T}x$)
  - g links natural parameter to canonical parameter
  - $g^{-1}$ links canonical parameter to natural parameter

- Logistic Regression
  > $h_{\theta}(x) = E[y|x;\theta] = \phi = \frac{1}{1+e^{-\eta}} = \frac{1}{1+e^{-\theta^{T}x}} $

- Are GLM used for classification/regression
  - depends on choice of distribution
  - GLM just a general way of model data which can be binary, real, exponential or others

- Assumptions
  - Regression
    - At every given x, there is a y given x which is Gaussian and is parameterized by $\theta^{T}x$ as mean
    - The assumption is there was a Gaussian distribution and you sampled the value from this Gaussian distribution
    - We assume that the data was generated as above and we will work backward to find $\theta$, which will give us boundary condition

$\tiny{\text{YouTube-Stanford-CS229-Andrew Ng/Anand Avati}}$

### Softmax Regression
- Cross entropy minimization
- Multiclass classification
- can work over thousand of classes
- Type of GLM
- one vector per class
- generalization of logistic regression, with different set of parameters per class
- Goal is to:
  - Start from this data, learn a model that can given a new data point, make a prediction of its class
- Notation
  - k - # of classes
  - $x^{(i)} \in \mathbb R^{n}$
  - Label: $y = [\{0,1\}^{k}]$
    - For example: [0, 0, 0, 1, 0] - assuming there are 5 class here
    - label is a vector which indicates which class the x corresponds to
    - each element in the vector corresponds to a vector
    - there will be only 1 in the label vector
- Each class has its own set of parameters
  - $\theta_{class} \in \mathbb R^{n}$

$\tiny{\text{YouTube-Stanford-CS229-Andrew Ng/Anand Avati}}$

- Example
  * For a given x, $\theta_{i}^{T}x$ (logit space) will have a range of $-\infty$ to $\infty$
  * goal is to get probability distribution over classes
    * inorder to do that, we exponentiate the logits ($exp(\theta_{i}^{T}x)$), which makes everything positive
    * then we normalize this, by the sum of all $\frac{e^{\theta_{i}^{T}x}}{\sum_{all\space classes} e^{\theta_{i}^{T}x}}$
    * this gives us a probability distribution $\hat{p}(y)$ over all the classes
    * minimize the distance between true label and learned label distribution using cross entropy

$\tiny{\text{YouTube-Stanford-CS229-Andrew Ng/Anand Avati}}$

* minimize the distance between these two distributions or minimize the cross entropy ($p, \hat p $)
  > Cross entropy($p, \hat p$)
  > $ = -\sum\limits_{y \in \{\Delta, \square, \circ \} } p(y) \text{log}\hat p(y)$
  > $ = - log\space\hat p(y_{\Delta})$ - associated class here is triangle
  > $ = - log \frac{e^{\theta_{\Delta}^{T}x}}{\sum_{c \in \{\Delta, \square, \circ \}} e^{\theta_{c}^{T}x}}$
* treat this as a loss and apply gradient descent wrt parameter


# definition
数值定义. 对于 N-bit two's complement number system, 最高位 N-th bit 为符号位, 0 为正,
1 为负. 对于任意一个非负整数, 它的相反数为 its complement with respect to $2^N$.

# properties

-   一个数字的 two's complement 可以通过:
    1.  take its ones' complement and add one.
        因为: the sum of a number and its ones' complement is -0, i.e. ‘1’ bits, or $2^N-1$;
        and by definition, the sum of a number and its two's complement is $2^N$.
    2.  通过定义中的 unsigned binary arithmetic 计算得出.
-   The value of A N-bit binary number in two's complement numercal system can be computed by
    $$w = -a_{N-1} 2^{N-1} + \sum_{i=0}^{N-2} a_i 2^i$$. 推导: 对于互为相反数的 $A$ 和 $B$,
    已知:
    $$
    \begin{align}
        A + B &= 0 \\
        A &= \sim B + 1
    \end{align}
    $$
    第二式为 two's complement 和 ones' complement 的关系.
    根据第二式得到
    $$
    A=1+\sum_{i=0}^{N-1}(1-b_i)2^i
    $$
    对于 N-bit number system, $A+B=0$ 意味着
    $$(A+B) \operatorname{mod} 2^N = 0$$
    满足上式的 $B$ 有两种设计方式:
    $$
    \begin{align}
        B &= b_{N-1}2^{N-1} + \sum_{i=0}^{N-2}b_i 2^i \\
        B &= -b_{N-1}2^{N-1} + \sum_{i=0}^{N-2}b_i 2^i
    \end{align}
    $$
    分别对应于 unsigned number 和 signed number 的数值公式.
-   most negative number, `INT_MIN`. 它没有可以在系统内表达的相反数. 因为正负数范围的不对称性.
    根据 two's complement 的定义计算 `INT_MIN` 得到的是自身, 这显然是不对的.
-   加减法. 普通的二进制加减法运算即可, 没有特殊处理.
-   加减法的溢出检测. an XOR operation on leftmost two carry/borrow bits can quickly determine
    if an overflow condition exists.

```python

```


```python
import fit_pendulum_data as p1
import midpoint_vec as p2
import Lagrange_poly1 as p3
import Lagrange_poly2 as p4
import Lagrange_poly2b as p5
import sympy as sp
import numpy as np
import matplotlib.pyplot as plt
```

# Exercise 5.18
## Plots L vs T from a source file and fits polynomials of varying degrees to it

```python
p1.part_a()
```

### Below are the polynomials being fit to the data

```python
p1.part_b()
```

# Exercise 5.22
## Computes the midpoint rule in different forms

```python
p2.midpointint(p2.function, 1, 3, 50)[0]
```

    12.231183999999999

```python
p2.sum_vectorized(p2.function, 1, 3, 50)
```

    12.231183999999999

```python
p2.sum_numpy(p2.function, 1, 3, 50)
```

    12.231183999999999

# Exercise 5.23, 5.24, 5.25 (3 part)
## Lagrange Interpolation

```python
p4.graph(p4.sin, 20, 0, 10, [0,10,-2,2])
```

```python
p5.problem_5_25()
```

```python

```


# COURSE: Master math by coding in Python
## SECTION: Algebra 2

#### https://www.udemy.com/course/math-with-python/?couponCode=MXC-DISC4ALL
#### INSTRUCTOR: sincxpress.com

Note about this code: Each video in this section of the course corresponds to a section of code below. Please note that this code roughly matches the code shown in the live recording, but is not exactly the same -- the variable names, order of lines, and parameters may be slightly different.

```python
import sympy as sym
import numpy as np
import scipy as sp
import matplotlib.pyplot as plt
from IPython.display import display, Math
```

# VIDEO: Summation and products

```python
lst = [1,3,4,1,6]

print( sum(lst) )
print( np.sum(lst) )

```

```python
np.prod(lst)
```

```python
# cumulative summation
np.cumsum(lst)

plt.plot(lst,'rs-')
plt.plot(np.cumsum(lst),'bo-')
plt.legend({'list','cumsum'})
plt.show()
```

### Exercise

```python
# generate numerator
num = np.arange(1,6)
den = num**2

ans1 = np.sum(num) / np.sum(den)
ans2 = 1 / np.sum(num)

print('Option 1 = %s\nOption 2 = %s' %(ans1,ans2))
```

```python
# then repeat with prod
ans1 = np.prod(num) / np.prod(den)
ans2 = 1 / np.prod(num)

print('Option 1 = %s\nOption 2 = %s' %(ans1,ans2))
```

# VIDEO: Differences (discrete derivative)

```python
print(2-1)
```

```python
v = [1,2]
np.diff(v)
```

```python
v = [1,2,4,-10]
np.diff(v)
```

```python
v = np.arange(0,11)
print(v)
print(np.diff(v))
print(np.diff(v,2))
print(np.diff(np.diff(v)))
```

### Exercise

```python
# plot x^2 and its derivative
x = np.linspace(-2,2,1021)
f = x**2
df = np.diff(f)
dx = np.diff(x)

plt.plot(x,f,label='f')
plt.plot(x[0:-1],df/dx,'r',label='df/dx')

plt.axis([-2,2,-1,2])
plt.grid()
plt.legend()
plt.show()
```

```python

```

# VIDEO: Roots of polynomials

```python
### the equation:
# 3x^2 + 2x - 1

# define the coefficients
coefs = [3,2,-1]

# and solve
roots = np.roots(coefs)

print(roots)
```

```python
# solve
from sympy.abc import x

eq = 3*x**2 + 2*x - 1

for i in roots:
    display(Math('\\text{At x=}%g\\text{, }%s = %g' %(i,sym.latex(eq),eq.subs(x,i))))

```

```python
# zero-degree polynomial
np.roots(8)
```

### Exercise

```python
# for different polynomial degrees, count the number of roots

for i in range(1,11):
    coefs = np.arange(1,i+1)
    print('A degree-%s polynomial has %s roots' %(len(coefs)-1,len(np.roots(coefs))))
```

# VIDEO: The quadratic equation and functions

```python
# the quadratic equation

a = 2
b = 7
c = 5

quadeqP = ( -b + np.sqrt(b**2 - 4*a*c) ) / (2*a)
quadeqM = ( -b - np.sqrt(b**2 - 4*a*c) ) / (2*a)

print(quadeqP,quadeqM)
```

```python
# you can use scipy instead of numpy for complex results to sqrt...

a = 3
b = 7
c = 5

quadeqP = ( -b + sp.sqrt(b**2 - 4*a*c) ) / (2*a)
quadeqM = ( -b - sp.sqrt(b**2 - 4*a*c) ) / (2*a)

print(quadeqP,quadeqM)
```

```python
# ... or explicitly tell numpy to allow complex-valued solutions

a = 3
b = 7
c = 5

quadeqP = ( -b + np.sqrt(b**2 - 4*a*c,dtype=complex) ) / (2*a)
quadeqM = ( -b - np.sqrt(b**2 - 4*a*c,dtype=complex) ) / (2*a)

print(quadeqP,quadeqM)
```

```python
# as a function!

def quadeq(a,b,c):

    # initialize
    out = sp.zeros(2)

    # compute
    out[0] = ( -b - sp.sqrt(b**2 - 4*a*c) ) / (2*a)
    out[1] = ( -b + sp.sqrt(b**2 - 4*a*c) ) / (2*a)

    # output
    return out
```

```python
# test!
print(quadeq(2,7,2))

print(quadeq(1,4,-2))
```

### Exercise

```python
a = 1
b = range(-5,6)
c = range(-2,11)

M = np.zeros((len(b),len(c)))

for i in range(0,len(b)):
    for j in range(0,len(c)):
        M[i,j] = quadeq(a,b[i],c[j])[0]

# NOTE: In the video the y-axis label is incorrect, because the image has a flipped orientation.
# adding origin='lower' into imshow will correct the problem. Sorry for the confusion!
plt.imshow(M,extent=[ c[0],c[-1],b[0],b[-1] ],origin='lower')
plt.xlabel('c')
plt.ylabel('b')
plt.title('a='+str(a))
plt.colorbar()
plt.show()

```

# VIDEO: Complex numbers: addition and subtraction

```python
# some correct and incorrect ways to get the imaginary operator

print( 1j )
print( np.sqrt(-1) ) # doesn't work!
print( np.sqrt(-1,dtype='complex') )
print( (-1)**(1/2) ) # OK but not a good solution
print(sym.I) # in sympy
```

```python
# create complex numbers

real_part = 4
imag_part = -6

cn1 = np.complex(real_part,imag_part)
cn2 = real_part + 1j*imag_part
```

```python
# display
print(cn1)
print(cn2)

print( type(cn2) )
```

```python
# now for adding and subtracting
z1 = np.complex(4,5)
z2 = np.complex(6,-2)

print(z1+z2)
print(z1-z2)

```

```python
# extracting the real and imaginary parts of a complex number
print( 'The real part of z is ' + str(np.real(z1)))
print( 'The imaginary part of z is ' + str(np.imag(z1)))
```

### Exercise

```python
# show that the complex addition is real parts summed and imaginary parts summed

w = np.complex(2,4)
z = np.complex(5,6)

# sum each part individually
realsum = np.real(w) + np.real(z)
imagsum = np.imag(w) + np.imag(z)

wPlusz1 = np.complex(realsum,imagsum)

# sum together in one go
wPlusz2 = w+z

print(wPlusz1)
print(wPlusz2)

```

# VIDEO: Complex numbers: conjugate and multiplication

```python
# create complex numbers
z1 = np.complex(4,5)
z2 = np.complex(6,-2)

# multiplication the "manual" way
w = np.real(z1)*np.real(z2) + np.real(z1)*np.imag(z2)*1j + np.imag(z1)*np.real(z2)*1j + np.imag(z1)*np.imag(z2)*1j*1j

print(w)
sym.sympify(w)

```

```python
# or let Python do the hard work ;)
sym.sympify(z1*z2)
```

```python
# complex conjugate

z1 = np.complex(4,3)
z2 = np.complex(4,-3)

print( np.conj(z1) )
print( np.conj(z2) )

```

### Exercise

```python
a,b = sym.symbols('a,b')#,real=True)

z = a + b*sym.I

sym.simplify( sym.expand( z*sym.conjugate(z) ) )
```

```python
# with nicer printing

result = sym.simplify( sym.expand( z*sym.conjugate(z) ) )
display(Math('z\\times z^* = %s' %sym.latex(result)))
```

# VIDEO: Complex numbers: division

```python
# create a complex number
z1 = np.complex(4,2)
z2 = np.complex(3,-5)

display(Math('\\frac{%s}{2} = %s' %(z1,z1/2) ))

```

```python
display(Math('\\frac{%s \\times %s}{%s \\times %s} = \\frac{%s}{%s} = %s' # sometimes you need \
             %(z1,np.conj(z2), z2, np.conj(z2),
               z1*np.conj(z2), z2*np.conj(z2), sym.latex(sym.sympify(z1/z2)) )))
```

### Exercises

```python
# create two numbers
z = np.complex(4,2)
w = np.complex(3,5)

# the parts related to the left multiplicand
left_numer = (z-1) * (z*w+w)
left_denom = w*z-w
left_part = left_numer / left_denom

# the parts related to the right multiplicand
right_numer = (w-1) * (1-w)
right_denom = -w**2*z - w**2 + 2*w*z + 2*w - z - 1
right_part = right_numer / right_denom

# put them together
solution = left_part * right_part

# and print!
print(solution)

```

# VIDEO: Graphing complex numbers

```python
# create a complex number
z = np.complex(2,3)

# not so simple...
plt.plot(z,'ro')
plt.show()

# how it's done
plt.plot(np.real(z),np.imag(z),'ro')
plt.xlabel('Real')
plt.ylabel('Imag')
plt.grid()
plt.axis([-4,4,-4,4])
plt.show()

```

```python
# drawing complex numbers as lines from the origin

# create complex numbers
z1 = np.complex(4,5)
z2 = np.complex(6,-2)

# draw the lines
plt.plot([0,np.real(z1)],[0,np.imag(z1)],color='b',label='z1')
plt.plot([0,np.real(z2)],[0,np.imag(z2)],'b--',label='z2')

# new variables for convenience
z1Pz2 = z1+z2
z1Mz2 = z1-z2

# draw more lines!
plt.plot([0,np.real(z1Pz2)],[0,np.imag(z1Pz2)],'r',label='z1+z2',linewidth=2)
plt.plot([0,np.real(z1Mz2)],[0,np.imag(z1Mz2)],'k',label='z1-z2',linewidth=2)

# make the plot look a bit nicer
plt.axis('square')
plt.axis(np.multiply([-1,1,-1,1],np.abs(z1Pz2)))
plt.grid(True)
plt.legend()
plt.show()
```

### Exercises

```python
# adding two complex numbers
z1 = np.complex(-3,1)
z2 = np.complex(-1,1)

# their sum
z3 = z1+z2

plt.plot([0,np.real(z1)],[0,np.imag(z1)],label='z1')
plt.plot([0,np.real(z2)],[0,np.imag(z2)],label='z2')
plt.plot([0,np.real(z3)],[0,np.imag(z3)],label='z1+z2')

plt.axis('square')
plt.axis([-5,5,-5,5])
plt.legend()
plt.grid()
plt.xlabel('Real')
plt.ylabel('Imag')

plt.show()
```

```python
# multiplying two complex numbers

# their product
z3 = z1*z2

plt.plot([0,np.real(z1)],[0,np.imag(z1)],label='z1')
plt.plot([0,np.real(z2)],[0,np.imag(z2)],label='z2')
plt.plot([0,np.real(z3)],[0,np.imag(z3)],label='$z1\\times z2$')

plt.axis('square')
plt.axis([-5,5,-5,5])
plt.legend()
plt.grid()
plt.xlabel('Real')
plt.ylabel('Imag')

plt.show()
```

# VIDEO: Revising the quadratic equation with complex numbers

```python
# as a function!

def quadeq(a,b,c):

    # initialize
    out = sp.zeros(2,dtype=complex)

    # compute
    out[0] = ( -b - sp.sqrt(b**2 - 4*a*c) ) / (2*a)
    out[1] = ( -b + sp.sqrt(b**2 - 4*a*c) ) / (2*a)

    # output
    return out
```

```python
x = 1
y = range(-5,6)
z = range(-2,11)

M = np.zeros((len(y),len(z)),dtype=complex)

for i in range(0,len(y)):
    for j in range(0,len(z)):
        M[i,j] = quadeq(1,y[i],z[j])[0]

plt.subplot(1,3,1)
plt.imshow(sp.real(M))
plt.axis('off'), plt.title('real part')

plt.subplot(132)
plt.imshow(sp.imag(M))
plt.axis('off'), plt.title('imaginary part')

plt.subplot(133)
plt.imshow(sp.absolute(M))
plt.axis('off'), plt.title('magnitude')

plt.show()
```

# VIDEO: The unit circle

```python
# create a unit circle via trigonometry

# define the phase angles
x = np.linspace(0,2*np.pi,100)

# generate the plot
plt.plot(np.cos(x),np.sin(x),'k')

# draw axis lines
plt.plot([-1.3,1.3],[0,0],'--',color=[.8,.8,.8])
plt.plot([0,0],[-1.3,1.3],'--',color=[.8,.8,.8])

# make it look nicer
plt.axis('square')
plt.axis([-1.3,1.3,-1.3,1.3])
plt.xlabel('cosine(x)')
plt.ylabel('sine(x)')
plt.plot()

plt.show()
```

```python
# draw the circle again
plt.plot(np.cos(x),np.sin(x),'k')

# now draw a vector
angle = np.pi/4
plt.plot([0,np.cos(angle)],[0,np.sin(angle)],'r',linewidth=3)

# make it look nicer
plt.axis('square')
plt.xlabel('cosine(x)')
plt.ylabel('sine(x)')
plt.axis([-1.3,1.3,-1.3,1.3])

plt.show()
```

### Exercises

```python
# something with log-space increases in angle
x = np.linspace(-3,1.83,50)
a = np.exp(x)

for i in a:
    plt.plot([0,np.cos(i)],[0,np.sin(i)])

# make it look nicer
plt.axis('square')
plt.axis('off')

plt.show()
```

# VIDEO: Natural exponent and logarithm

```python
# e^x

x = np.linspace(-3,3,100)

plt.plot(x,np.exp(x))
plt.xlabel('x')
plt.ylabel('$e^x$')
plt.show()

```

```python

plt.plot(x,np.log(x))
plt.xlabel('x')
plt.ylabel('log(x)')
plt.show()

```

### Exercise

```python
# visualize that log and e are inverses of each other (positive values only)

x = np.linspace(.0001,10,10)

# plot it both ways
plt.plot(x,np.log(np.exp(x)))
plt.plot(x,np.exp(np.log(x)),'o')

plt.axis('square')
plt.legend({'$\\log(e^x)$','$e^{\\log(x)}$'})
plt.show()

```

# VIDEO: Find a specific point on a Gaussian

```python
# the key parameter (FWHM - full width at half-maximum)
h = 1

# "Time" variable
t = np.linspace(-3,3,1001)

# gaussian
g = np.exp( -4*np.log(2)*t**2 / h**2 )

# and plot
plt.plot(t,g)
plt.xlabel('t')
plt.ylabel('y')
plt.title('Gaussian with FWHM=' + str(h))
plt.show()

```

```python
# find the x-axis value (here variable 't') for a given function value

yVal2find = .5

# apply the formula shown in the slides
tvals = h*np.sqrt( np.log(yVal2find)/(-4*np.log(2)) )

# print out the results
print(tvals,-tvals)
```

### Exercise

```python
N = 100 # number of Gaussians
h = np.zeros(N)

G = np.zeros((N,len(t)))

for i in range(N):
    h[i] = (i+1)/N
    G[i,:] = np.exp( -4*np.log(2)*t**2 / h[i]**2 )

plt.pcolormesh(t,h,G)
plt.xlabel('t')
plt.ylabel('h')
plt.show()
```

# VIDEO: Graphing the complex roots of unity

```python
# confirm that the roots of unity produce 1

n = 5
for k in range(0,n):
    root = sym.exp( (2*k*sym.pi*sym.I)/n )
    display(Math('%s \\Rightarrow %s' %(sym.latex(root),sym.latex(root**n)   )))

# print(roots)
# print(np.real(np.prod(roots)))
```

```python
# parameter for number of roots to show
n = 6

for k in range(0,n):

    # compute this root
    root = np.exp( (2*k*np.pi*1j)/n )

    # plot it
    plt.plot([0,np.real(root)],[0,np.imag(root)],linewidth=3)

# generate the plot with the unit circle on top
x = np.linspace(0,2*np.pi,100) # copied from previous lecture!
plt.plot(np.cos(x),np.sin(x),color='gray')

# draw axis lines
plt.plot([-1.3,1.3],[0,0],'--',color='gray')
plt.plot([0,0],[-1.3,1.3],'--',color='gray')

# make the plot look nicer
plt.axis([-1.3,1.3,-1.3,1.3])
plt.axis('square')
plt.axis('off')
plt.title('The %g %gth roots of 1' %(n,n))
plt.show()
```

### Exercise

```python
# parameter for number of roots to show
n = 200

color = np.linspace(0,.9,n)

for k in range(0,n):

    # compute this root
    root = k*np.exp( (2*k*np.pi*1j)/n )

    # plot it
    plt.plot([0,np.real(root)],[0,np.imag(root)],linewidth=2,color=[color[k],color[k],color[k]])

# make the plot look nicer
plt.axis('square')
plt.axis('off')
plt.show()
```

# VIDEO: Log-spaced and linearly spaced numbers

```python
# you've already seen linearly spaced numbers...

np.linspace(-2,4,10)
```

```python
# log-spaced
np.logspace(1,2,10)
```

```python
# the way you'd probably expect them to be
a = np.log10(2.3)
b = np.log10(14.3)
n = 14

np.logspace(a,b,n)
```

```python
# the way you'd probably expect them to be
a = np.log10(1)
b = np.log10(5)
n = 5

np.logspace(a,b,n)
```

### Exercise

```python
# show that at small scales, lin and log are close.
# this is why local linear approximations to nonlinear functions are so useful

a = 2
b = 100
n = 50

lo = np.logspace(np.log10(a),np.log10(b),n)
li = np.linspace(a,b,n)

plt.plot(li,lo,label='log')
plt.plot(li,li,label='linear')

plt.legend()
plt.axis('square')
plt.show()

```

# VIDEO: Logarithm properties: Multiplication and division

```python
a = 3
b = 4

# test the multiplication rule

res1 = np.log(a*b)
res2 = np.log(a) * np.log(b)
res3 = np.log(a) + np.log(b)

display(Math('\\log(%g\\times%g) = %g' %(a,b,res1)))
display(Math('\\log(%g)\\times\\log(%g) = %g' %(a,b,res2)))
display(Math('\\log(%g)+\\log(%g) = %g' %(a,b,res3)))

display(Math('\\log(%g\\times%g) \\neq \\log(%g)\\times\\log(%g)' %(a,b,a,b)))
display(Math('\\log(%g\\times%g) = \\log(%g)+\\log(%g)' %(a,b,a,b)))

```

```python
# repeat for division

res1 = np.log(a/b)
res2 = np.log(a) / np.log(b)
res3 = np.log(a) - np.log(b)

display(Math('\\log(\\frac{%g}{%g}) = %g' %(a,b,res1)))
display(Math('\\log(%g)/\\log(%g) = %g' %(a,b,res2)))
display(Math('\\log(%g)-\\log(%g) = %g' %(a,b,res3)))

display(Math('\\log{\\left(\\frac{%g}{%g}\\right)} \\neq \\frac{\\log(%g)}{\\log(%g)}' %(a,b,a,b)))
display(Math('\\log{\\left(\\frac{%g}{%g}\\right)} = \\log(%g)-\\log(%g)' %(a,b,a,b)))

```

### Exercise

```python
# exponential rule

expr1 = np.log(a**b)
expr2 = b*np.log(a)

print(expr1-expr2)
```

```python
expr1 = np.log(3**3)
expr2 = 3*np.log(3)
expr3 = np.log(3) + np.log(3) + np.log(3)

print(expr1,expr1-expr2,expr1-expr3)
```

# VIDEO: Scientific notation

```python
# write the following number in scientific notation
x = 2938429348

'{:,e}'.format(x)
```

```python
# numbers are easier to read with commas!

# or f
'{:,d}'.format(x)
```

```python
# another way...
'%.2e' %(x)
```

```python
# notice this is a string

s = '%.3e' %(x)
print(s)
print( type(s) )

# so you can index or slice parts of it
s[2:4]
```

### Exercise

```python
# list the number
num = 2342

# order of magnitude
orderOfMag = int(np.floor(np.log10(abs(num))))

# convert to scientific notation and get the characters before the period
numstr = '{:,e}'.format(num)
wheredot = numstr.find('.')
scinot = numstr[:wheredot]

display(Math( '%s\\text{ is }%s\\text{ orders of magnitude, }\\approx %s\\times10^{%g}' \
             %(num,orderOfMag,scinot,orderOfMag) ))
```

# VIDEO: Maxima and minima of a function

```python
# max and argmax
lst = [-4,5.5,3,2]

print(np.max(lst))
print(np.argmax(lst))

```

```python
# method: evaluate specific points in the function

# x range
x = np.linspace(0,2*np.pi,20)

# the function f(x)
fx = -( np.cos(x) + x**(1/2) )

# find the maximum value of this function
fmax = np.max(fx)
print(fmax)

# but we also need to know the location where this maximum is obtained
fmaxidx = np.argmax(fx)
print(fmaxidx)

# draw the function
plt.plot(x,fx,'bo-')
plt.plot(x[fmaxidx],fmax,'rs')

plt.xlabel('radians')
plt.xticks(np.round(np.arange(0,2*np.pi,np.pi/4),2))
plt.show()
```

```python
# minimum

# the range and the function
x = np.linspace(-2,2,50)
fx = x**3 + x**4

# find the minimum
fminidx = np.argmin(fx)

# draw the function
plt.plot(x,fx,'bo-')
plt.plot(x[fminidx],fx[fminidx],'ms',markersize=10)

plt.ylim([-.2,.2])
plt.xlim([-1.2,.5])
plt.show()

```

### Exercise

```python
from scipy.signal import find_peaks

# x range
x = np.linspace(0,12*np.pi,200)

# the function f(x)
fx = -( np.cos(x) + x**(1/2) )

# find peaks
peeks = find_peaks(fx)
print(peeks)

# draw the function
plt.plot(x,fx)
plt.plot(x[peeks[0]],fx[peeks[0]],'o')
plt.show()

```

# VIDEO: Even and odd functions

```python
# create an even function
x = np.linspace(-5,5,20)

fEven = x**2

fEvenNegative = (-x)**2

plt.plot(x,fEven)
plt.plot(x,fEvenNegative,'ro')
plt.show()
```

```python
# create an odd function
x = np.linspace(-5,5,20)

fOdd = x**3

fOddNegative = (-x)**3

plt.plot(x,fOdd)
plt.plot(x,fOddNegative,'ro')
plt.plot(x,-fOddNegative,'gs')
plt.show()
```

### Exercise

```python
# using graphing to infer whether cos and sine are even or odd functions

theta = np.linspace(-2*np.pi,2*np.pi,40)

cosfun = np.cos(theta)
cosfunN = np.cos(-theta)

plt.plot(theta,cosfun)
plt.plot(theta,cosfunN,'ro')
plt.show()

# repeat for sin
sinfun = np.sin(theta)
sinfunN = np.sin(-theta)

plt.plot(theta,sinfun)
plt.plot(theta,sinfunN,'ro')
plt.plot(theta,-sinfunN,'ys')
plt.show()

```

# VIDEO: Arithmetic and geometric sequences

```python
# arithmetic sequence

a = 2
d = 3
maxn = 6

ariseq = a + d*np.arange(0,maxn)
print(ariseq)
```

```python
# geometric sequence
a = 2
r = 3

geoseq = a * r**np.arange(0,maxn)
print(geoseq)

```

```python
plt.plot(ariseq,'ks',label='arithmetic')
plt.plot(geoseq,'ro',label='geometric')

plt.xlim([-.5,maxn-.5])
plt.legend()
plt.show()
```

### Exercise

```python
# confirm that nth term is a+(n-1)*d and a*r**(n-1) by indexing and direct computation

# parameters
a = 2
d = 3
maxn = 10

# compute sequences
ariseq = a + d*np.arange(0,maxn)
geoseq = a * d**np.arange(0,maxn)

# which element to find?
nth = 6

# find by direct computation
ariDirect = a + (nth-1)*d
geoDirect = a * d**(nth-1)

# compare against order in sequence
print(ariseq[nth-1],ariDirect)
print(geoseq[nth-1],geoDirect)

```

# VIDEO: Algebra BUG HUNT!

```python
# create a list
lst = [1,3,4,1,6]

print( lst )
```

```python
# add all the numbers together
print( np.sum(lst) )
```

```python
# plot the cumulative sum of a list of numbers
l = np.arange(-4,10)

plt.plot(l,'rs-')
plt.plot(np.cumsum(1),'bo-')
plt.legend({'list','cumsum'})
plt.show()
```

```python
### the equation:
# 4 - 2x + 5x^3

# define the coefficients
coefs = [5,0,-2,4]

# solve
roots = np.roots(coefs)

# and display
from sympy.abc import x

p = sym.Poly(coefs,x)

display(Math(sym.latex(p)))
```

```python
def quadeq(a,b,c):

    # compute
    out0 = ( -b - np.sqrt(b**2 - 4*a*c,dtype=complex) ) / (2*a)
    out1 = ( -b + np.sqrt(b**2 - 4*a*c,dtype=complex) ) / (2*a)

    # output
    return [out0,out1]

# test the function
print(quadeq(1,4,2))
```

```python
# create a complex number

real_part = 4
imag_part = -6

cn = np.complex(real_part,imag_part)

plt.plot(np.real(cn),np.imag(cn),'ro')
plt.grid('on')
plt.axis('square')
plt.axis([-10,10,-10,10])
plt.show()
```

```python
# symbolic complex number
a,b = sym.symbols('a,b',real=True)

z = a + b*sym.I
display(Math('z\\times z^* = %s' %sym.latex(sym.expand(z*sym.conjugate(z)))))
```

```python
# define the phase angles
x = np.linspace(0,2*np.pi,100)

# generate the plot
plt.plot(np.cos(x),np.sin(x),'k')

# draw one vector from the origin
phs = 1*np.pi/4
plt.plot([0,np.cos(phs)],[0,np.sin(phs)],'r-')
plt.plot(np.cos(phs),np.sin(phs),'ro')

# draw axis lines
plt.plot([-1.3,1.3],[0,0],'--',color=[.8,.8,.8])
plt.plot([0,0],[-1.3,1.3],'--',color=[.8,.8,.8])

# make it look nicer
plt.axis('square')
plt.axis([-1.3,1.3,-1.3,1.3])
plt.xlabel('cosine(x)')
plt.ylabel('sine(x)')
plt.plot()

plt.show()
```

```python
a =   2 # lower bound
b = 100 # upper bound
n =   3 # number of steps

lo = np.logspace(np.log10(a),np.log10(b),n)
li = np.linspace(a,b,n)

plt.plot(li,lo,'s-',label='log')
plt.plot(li,li,'o-',label='linear')

plt.legend()
plt.axis('square')
plt.show()
```

```python
## Goal is to plot the point on the function closest to f(x)=.5

# x range
x = np.linspace(-6,10,20)

# the function f(x)
fx = 1/(1+np.exp(x))

# function maximum
fmaxidx = np.argmin(abs(fx-.5))

# draw the function
plt.plot(x,fx,'bo-')
plt.plot(x[fmaxidx],fx[fmaxidx],'rs')

plt.show()
```

```python
## Goal is to find the local minima by adding one character

from scipy.signal import find_peaks

# x range
x = np.linspace(0,12*np.pi,200)

# the function f(x)
fx = -( np.cos(x) + x**(1/2) )

# find peaks
peeks = find_peaks(-fx)

# draw the function
plt.plot(x,fx)
plt.plot(x[peeks[0]],fx[peeks[0]],'o')
plt.show()

```


# Initail Value Problem
> ## Given $\frac{dy}{dx} = 8x + csc^2(x)$ with $y(\pi/2) = -7$ solve for $y(x)$

```python
import sympy as sm
x, C = sm.symbols('x C')
sol = sm.integrate(8*x + sm.csc(x)**2, x)

# y = sol + C
# y(pi/2) = -7

C = -sol - 7
ans = sol + C.subs(x,sm.pi/2)
ans
# ans.subs(x,sm.pi/2)
```

$\displaystyle 4 x^{2} - \pi^{2} - 7 - \frac{\cos{\left(x \right)}}{\sin{\left(x \right)}}$

> ## $$ \int \frac{(1+\sqrt{x})^{1/3}}{\sqrt{x}}dx$$

```python
(1 + sm.sqrt(x))**sm.Rational(1,3) / sm.sqrt(x)
```

$\displaystyle \frac{\sqrt[3]{\sqrt{x} + 1}}{\sqrt{x}}$

> ## $$
\int x(1-x^2)^{1/4} dx
$$

```python
(x*(1-x**2))**sm.Rational(1/4)
```

$\displaystyle \sqrt[4]{x \left(1 - x^{2}\right)}$

# Definite Integrals
> ## $$ \int_0^{ln(4)}\frac{e^x}{e^{2x}+9}dt $$

```python
sm.exp(x) / (sm.exp(2*x) + 9)
```

$\displaystyle \frac{e^{x}}{e^{2 x} + 9}$

> ## $$\int _1^t x^{10}e^x dx $$

```python
t = sm.symbols('t')
sm.integrate(x**10 * sm.exp(x),(x,1,t))
```

$\displaystyle \left(t^{10} - 10 t^{9} + 90 t^{8} - 720 t^{7} + 5040 t^{6} - 30240 t^{5} + 151200 t^{4} - 604800 t^{3} + 1814400 t^{2} - 3628800 t + 3628800\right) e^{t} - 1334961 e$

# Improper Integrals
> ## $$ \int_0^{\infty} \frac{16tan^-1(x)}{1+x^2}dx$$

```python
sm.integrate(16*sm.atan(x) / (1+x**2),(x,0,sm.oo))
```

$\displaystyle 2 \pi^{2}$

```python

```