# *Thermal Transport in a 2D Ti:Sapphire Crystal*

In this demo, we solve the two-dimensional diffusion equation in a rectangular Ti:sapphire crystal, with Dirichlet boundary condition $u_D$
on the sides and initial condition $T_\mathrm{init}$ inside the crystal body.

$$
\begin{align}
  u'   &= \alpha\nabla^2 u        \quad\text{in a short cylinder} \\
  u    &= u_D  \hphantom{Du}\quad\text{on the side boundaries} \\
  u    &= u_0  \hphantom{Du}\quad\;\text{at $t = 0$}
\end{align}
$$
with boundary conditions
$$
\begin{align}
  u_0  &= T_0 + \exp\bigl[ -\tfrac{1}{2}\bigl(\tfrac{x}{\xi}\bigr)^{2p} \bigr]
                \bigl[ \tfrac{1}{2}(dT_\mathrm{lo} + dT_\mathrm{hi}) + (dT_\mathrm{hi} - dT_\mathrm{lo}) \bigl(\cos^2\bigl(\tfrac{\pi y}{L}\bigr) - \tfrac{1}{2} \bigr) \bigr], \\
  \text{or}\quad
  u_0  &= T_0 + \delta_T \exp\bigl[ -\tfrac{1}{2}\bigl(\tfrac{x}{\xi}\bigr)^{2p} \bigr]
                [\exp(-y/\lambda) + \exp((y-l)/\lambda)], \\
  u_D  &= T_0.
\end{align}
$$

## Preamble

Needed imports and definitions:

```python
%matplotlib inline
import matplotlib.pyplot as plt
import numpy as np
import time as tm
import copy

# import os
# import sys
# import re
# from IPython.display import Image
# from IPython.core.interactiveshell import InteractiveShell
# InteractiveShell.ast_node_interactivity = "all"
```

```python
from fenics import *
```

```python
def minmax(it):
    min = max = None
    for val in it:
        if min is None or val < min:
            min = val
        if max is None or val > max:
            max = val
    return min, max
```

simulation parameters:

```python
## define main simulation parameters
# -- crystal properties
a_Al2O3 = 5.65e-2    # cm^2/s diffusion constant of sapphire (Al2O3)
a_perp  = 5.58e-2    # perpendicular to C
a_para  = 5.77e-2    # parallel to C
# -- crystal dimensions
width = 10.0         # diameter /cm
leng =   7.60        # length /cm
# -- mesh density
nx = 75              # mesh density along x-axis
ny = 60              # mesh density along y-axis
# -- time step
Ttot =   1.00        # total simulation time
n_steps =  500       # number of time steps
dt = Ttot / n_steps  # size of time step
nip =  25            # number of intervals between plots

# derived parameters
hwd = width / 2

print("duration:  %5.3f" % Ttot)
print("time-step: %5.3f" % dt)
print("plot-step: %5.3f" % (dt * nip))
print("num-plots: %5d" % (n_steps / nip))
```

    duration:  1.000
    time-step: 0.002
    plot-step: 0.050
    num-plots:    20

mesh the domain:

```python
# create mesh and define function space
mesh = RectangleMesh(Point(-hwd, 0), Point(hwd, leng), nx, ny)
V = FunctionSpace(mesh, 'P', 1)

# ranges
xmin, xmax = minmax(mesh.coordinates()[:,0])
ymin, ymax = minmax(mesh.coordinates()[:,1])
print(xmin, xmax)
print(ymin, ymax)
```

    -5.0 5.0
    0.0 7.6

set boundary condition and initial value

```python
# define boundary and initial conditions, and sources
T0    = 20.0   # deg C
dT    =  3.80 # deg C
dThi  =  5.000 # deg C
dTlo  =  4.000 # deg C
wdT   =  1.00  # cm
sg_px = 8      # 1
sg_py = 1      # 1

# define Dirichlet boundary condition for sides
tol = 1.e-13
def boundary_D(x, on_boundary):
    return on_boundary and (near(x[0], -hwd, tol) or near(x[0], hwd, tol))
bc = DirichletBC(V, Constant(T0), boundary_D)

# define Neumann input, -∂u/∂n = g
# use small dT, say 0.009, to simulate heat deposition inwards, across the crystal faces
gi = Expression('-dT * exp( - pow(pow(x[0],2) / (2 * pow(wdT,2)), px))',
                degree=1, T0=T0, dT=dT, wdT=wdT, px=sg_px)

# define heat source
# could be used as a steady source, in which case set small values for dTlo and dThi (say, 0.012 and 0.039)
# or simply as an initial condition, in which case set much larger values for dTlo and dThi (say, 12.00 and 4.00)
#
# cos2 x complexSG
hsc = Expression('T0 + ((dThi+dTlo)/2 + (dThi-dTlo)*(pow(cos(pi*x[1]/l), 2*py) - 1/2)) \
                    * exp(-0.5 * pow(x[0] / (w*3/4 + (w/2) * (pow(cos(pi * x[1] / l), 2) - 1/2)), 2*px))',
                 degree=1, T0=T0, dThi=dThi, dTlo=dTlo, w=wdT, l=leng, px=sg_px, py=sg_py)
#
# cos2 x SG
hsg = Expression('T0 + ((dThi+dTlo)/2 + (dThi-dTlo)*(pow(cos(pi*x[1]/l), 2*py) - 1/2)) \
                    * exp(-0.5 * pow(x[0]/w, 2*px))',
                 degree=1, T0=T0, dThi=dThi, dTlo=dTlo, w=wdT, l=leng, px=sg_px, py=sg_py)
#
# exp(decay) x SG
hse = Expression('T0 + dT * (exp(-x[1]/dl) + exp((x[1]-l)/dl)) \
                    * exp(-0.5 * pow(x[0]/w, 2*px))',
                 degree=1, T0=T0, dT=dT, w=wdT, l=leng, dl=0.8*leng, px=sg_px)

# define initial value
# u_n = interpolate(Constant(T0), V)
u_n = interpolate(hse, V)
u_min = u_n(leng/2,hwd)
u_max = u_n(0.,0.)
print(u_min, u_max)
```

    20.0 24.888718228068722

define the variational problem and main simulation function `evolve`:

```python
# define variational problem
u = TrialFunction(V)
v = TestFunction(V)

# source term
f = Constant(0)  # no source term
# f = hs

# F = u*v*dx + dt*a_Al2O3*dot(grad(u), grad(v))*dx - (u_n + dt*f)*v*dx + gi*v*ds  # w/ Neumann BC
F = u*v*dx + dt*a_Al2O3*dot(grad(u), grad(v))*dx - (u_n + dt*f)*v*dx  # w/ heat deposition
a, L = lhs(F), rhs(F)

# define time-evolution function
def evolve():

    # report initial state
    yield u_n

    # time-stepping
    u = Function(V)
    t = 0
    for n in range(1, n_steps + 1):

        # update current time
        t += dt

        # compute solution
        solve(a == L, u, bc)

        # report current state
        if n % nip == 0:
            yield u

        # update previous solution
        u_n.assign(u)
```

**NB**: The next three cells should not be executed in sequence.
You must first reëxecute the previous two code cells before running
any one of the following three cells.

```python
n_rows = 5
n_cols = 5
fig_wd = 18
# default sizing here yields unit aspect ratio
plt.figure(figsize = (fig_wd, fig_wd * n_rows // n_cols))

idx = 0
for u in evolve():
    idx += 1
    plt.subplot(n_rows, n_cols, idx)
    plot(u, vmin=u_min, vmax=u_max)
```

```python
# default sizing here yields unit aspect ratio
plt.figure(figsize = (8,5))

tol = 3.e-5  # avoid hitting points outside the domain
xv = np.linspace(xmin * (1 - tol), xmax * (1 - tol), 51)
yv = np.linspace(ymin * (1 - tol), ymax * (1 - tol), 51)
ptsX = [(x_, leng/30) for x_ in xv]
ptsY = [(0., y_) for y_ in yv]

idx = 0
for u in evolve():
    idx += 1
    ux = np.array([u(pt) for pt in ptsX])
    plt.plot(xv, ux, lw=1)
plt.plot(xv, ux, 'k', lw=1)
plt.xlabel(r'transverse position $y\,/\,\mathrm{cm}$')
plt.ylabel(r'$T\,/\,{}^\circ\mathrm{C}$')
plt.show()
```

```python
# default sizing here yields unit aspect ratio
plt.figure(figsize = (8,5))

tol = 3.e-5  # avoid hitting points outside the domain
xv = np.linspace(xmin * (1 - tol), xmax * (1 - tol), 51)
yv = np.linspace(ymin * (1 - tol), ymax * (1 - tol), 51)
ptsX = [(x_, leng/30) for x_ in xv]
ptsY = [(0., y_) for y_ in yv]

idx = 0
for u in evolve():
    idx += 1
    uy = np.array([u(pt) for pt in ptsY])
    plt.plot(yv, uy, lw=1)
plt.plot(yv, uy, 'k', lw=1)
plt.xlabel(r'longitudinal position $y\,/\,\mathrm{cm}$')
plt.ylabel(r'$T\,/\,{}^\circ\mathrm{C}$')
plt.show()
```

```python
mesh
```


```python
import numpy as np
import sympy as sym
import pydae.build as db
```

```python
params_dict = {'R_w':0.316,'G':9.81,'M':1200.0,'K_w':1.0,'C_rr':0.03,'Rho':1.225,'S_f':2.13,'C_x':0.32,'K_sign':100}

u_ini_dict = {'tau_r':0.0,'beta':0.0}  # for the initialization problem
u_run_dict = {'tau_r':0.0,'beta':0.0}  # for the running problem (here initialization and running problem are the same)

x_list = ['nu','x_pos']    # [inductor current, PI integrator]
y_ini_list = ['omega_r','snu'] # for the initialization problem
y_run_list = ['omega_r','snu'] # for the running problem (here initialization and running problem are the same)

sys_vars = {'params':params_dict,
            'u_list':u_run_dict,
            'x_list':x_list,
            'y_list':y_run_list}

exec(db.sym_gen_str())  # exec to generate the required symbolic varables and constants
```

```python
tau_w = tau_r/K_w
f_w = tau_w/R_w
f_b = G*M*sym.sin(beta)
f_d = 0.5*Rho*S_f*C_x*nu**2*snu
f_r = C_rr*G*M*snu
p_r = tau_r*omega_r

eq_snu = -snu + (1/(1+sym.exp(-nu*K_sign)))*2-1
eq_omega_r =-omega_r +  nu/(K_w*R_w)
dnu = 1/M*(f_w - f_d - f_r - f_b)
dx_pos = nu

```

```python
f_list = [dnu,dx_pos]
g_list = [eq_snu,eq_omega_r]
h_dict = {'f_w':f_w,'f_d':f_d,'f_r':f_r,'tau_r':tau_r,'p_r':p_r}

sys = {'name':f"mech",
       'params_dict':params_dict,
       'f_list':f_list,
       'g_list':g_list,
       'x_list':x_list,
       'y_ini_list':y_list,
       'y_run_list':y_list,
       'u_run_dict':u_run_dict,
       'u_ini_dict':u_ini_dict,
       'h_dict':h_dict
      }

sys = db.system(sys)
db.sys2num(sys)
```

```python
eq_snu
```

$\displaystyle - snu - 1 + \frac{2}{1 + e^{- K_{sign} \nu}}$

```python

```


# Linear Gaussian filtering and smoothing (continuous-discrete)

Provided is an example of linear state-space models on which one can perform Bayesian filtering and smoothing in order to obtain
a posterior distribution over a latent state trajectory based on noisy observations.
In order to understand the theory behind these methods in detail we refer to [1] and [2].

**References**:
> [1] Särkkä, Simo, and Solin, Arno. Applied Stochastic Differential Equations. Cambridge University Press, 2019.
>
> [2] Särkkä, Simo. Bayesian Filtering and Smoothing. Cambridge University Press, 2013.

```python
import numpy as np

import probnum as pn
from probnum import filtsmooth, randvars, randprocs
from probnum.problems import TimeSeriesRegressionProblem
```

```python
rng = np.random.default_rng(seed=123)
```

```python
# Make inline plots vector graphics instead of raster graphics
%matplotlib inline
from IPython.display import set_matplotlib_formats

set_matplotlib_formats("pdf", "svg")

# Plotting
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec

plt.style.use("../../probnum.mplstyle")
```

    /tmp/ipykernel_125705/236124620.py:5: DeprecationWarning: `set_matplotlib_formats` is deprecated since IPython 7.23, directly use `matplotlib_inline.backend_inline.set_matplotlib_formats()`
      set_matplotlib_formats("pdf", "svg")

## **Linear Continuous-Discrete** State-Space Model: Ornstein-Uhlenbeck Process

Now, we have a look at **continuous** dynamics. We assume that there is a continuous process that defines the dynamics of our latent space from which we collect discrete linear-Gaussian measurements (as above). Only the dynamics model becomes continuous. In particular, we formulate the dynamics as a stochastic process in terms of a linear time-invariant stochastic differential equation (LTISDE). We refer to [1] for more details.
Consider matrices $\boldsymbol{F} \in \mathbb{R}^{d \times d}$, $\boldsymbol{L} \in \mathbb{R}^{s \times d}$ and $H \in \mathbb{R}^{m \times d}$ where $d$ is the state dimension and $m$ is the dimension of the measurements.
We define the following **continuous-discrete** state-space model:

Let $x(t_0) \sim \mathcal{N}(\mu_0, \Sigma_0)$.

$$
\begin{align}
    d\boldsymbol{x} &= \boldsymbol{F} \, \boldsymbol{x} \, dt + \boldsymbol{L} \, d \boldsymbol{\omega} \\
    \boldsymbol{y}_k &\sim \mathcal{N}(\boldsymbol{H} \, \boldsymbol{x}(t_k), \boldsymbol{R}), \qquad k = 1, \dots, K
\end{align}
$$

where $\boldsymbol{\omega} \in \mathbb{R}^s$ denotes a vector of driving forces (often Brownian Motion).

Note that this can be generalized to a linear time-varying state-space model, as well. Then $\boldsymbol{F}$ is a function $\mathbb{T} \rightarrow \mathbb{R}^{d \times d}$,
$\boldsymbol{L}$ is a function $\mathbb{T} \rightarrow \mathbb{R}^{s \times d}$, and $H$ is a function $\mathbb{T} \rightarrow \mathbb{R}^{m \times d}$ where $\mathbb{T}$ is the "time dimension". In the following example, however, we consider a LTI SDE, namely, the Ornstein-Uhlenbeck Process from which we observe discrete linear Gaussian measurements.

### Define State-Space Model

#### I. Continuous Dynamics Model: Linear, Time-Invariant Stochastic Differential Equation (LTISDE)

```python
state_dim = 1
observation_dim = 1
```

```python
delta_t = 0.2
# Define Linear, time-invariant Stochastic Differential Equation that models
# the (scalar) Ornstein-Uhlenbeck Process
drift_constant = 0.21
dispersion_constant = np.sqrt(0.5)
drift = -drift_constant * np.eye(state_dim)
force = np.zeros(state_dim)
dispersion = dispersion_constant * np.eye(state_dim)
```

The _continuous_ counterpart to the discrete LTI Gaussian model is provided via the `LTISDE` class.

```python
# Create dynamics model
dynamics_model = randprocs.markov.continuous.LTISDE(
    drift_matrix=drift,
    force_vector=force,
    dispersion_matrix=dispersion,
)
```

#### II. Discrete Measurement Model: Linear, Time-Invariant Gaussian Measurements

```python
measurement_marginal_variance = 0.1
measurement_matrix = np.eye(observation_dim, state_dim)
measurement_noise_matrix = measurement_marginal_variance * np.eye(observation_dim)
```

As above, the measurement model is discrete, LTI Gaussian. Only the dymanics are continuous (i.e. continuous-discrete).

```python
noise = randvars.Normal(mean=np.zeros(observation_dim), cov=measurement_noise_matrix)
measurement_model = randprocs.markov.discrete.LTIGaussian(
    transition_matrix=measurement_matrix,
    noise=noise,
)
```

#### III. Initial State Random Variable

```python
mu_0 = 10.0 * np.ones(state_dim)
sigma_0 = np.eye(state_dim)
initial_state_rv = randvars.Normal(mean=mu_0, cov=sigma_0)
```

```python
prior_process = randprocs.markov.MarkovProcess(
    transition=dynamics_model, initrv=initial_state_rv, initarg=0.0
)
```

### Generate Data for the State-Space Model

Next, sample both latent states and noisy observations from the specified state-space model.

```python
time_grid = np.arange(0.0, 10.0, step=delta_t)
```

```python
latent_states, observations = randprocs.markov.utils.generate_artificial_measurements(
    rng=rng,
    prior_process=prior_process,
    measmod=measurement_model,
    times=time_grid,
)
```

```python
regression_problem = TimeSeriesRegressionProblem(
    observations=observations,
    locations=time_grid,
    measurement_models=[measurement_model] * len(time_grid),
)
```

### Kalman Filtering

In fact, since we still consider a **linear** model, we can apply Kalman Filtering in this case again.
According to Section 10 in [1], the moments of the filtering posterior in the continuous-discrete case are solutions to linear differential equations, which `probnum` solves for us when invoking the `<Kalman_object>.filtsmooth(...)` method.

#### I. Kalman Filter

```python
kalman_filter = filtsmooth.gaussian.Kalman(prior_process)
```

#### II. Perform Kalman Filtering + Rauch-Tung-Striebel Smoothing

```python
state_posterior, _ = kalman_filter.filtsmooth(regression_problem)
```

The method `filtsmooth` returns a `KalmanPosterior` object which provides convenience functions for e.g. sampling and prediction.
We can also extract the just computed posterior smoothing state variables by querying the `.state_rvs` property.
This yields a list of Gaussian Random Variables from which we can extract the statistics in order to visualize them.

```python
grid = np.linspace(0, 11, 500)

posterior_state_rvs = state_posterior(
    grid
)  # List of <num_time_points> Normal Random Variables
posterior_state_means = posterior_state_rvs.mean.squeeze()  # Shape: (num_time_points, )
posterior_state_covs = posterior_state_rvs.cov  # Shape: (num_time_points, )

samples = state_posterior.sample(rng=rng, size=3, t=grid)
```

### Visualize Results

```python
state_fig = plt.figure()

ax = state_fig.add_subplot()

# Plot means
ax.plot(grid, posterior_state_means, label="posterior mean")

# Plot samples
for smp in samples:
    ax.plot(
        grid,
        smp[:, 0],
        color="gray",
        alpha=0.75,
        linewidth=1,
        linestyle="dashed",
        label="sample",
    )

# Plot marginal standard deviations
std_x = np.sqrt(np.abs(posterior_state_covs)).squeeze()
ax.fill_between(
    grid,
    posterior_state_means - 1.96 * std_x,
    posterior_state_means + 1.96 * std_x,
    alpha=0.2,
    label="1.96 marginal stddev",
)
ax.scatter(time_grid, observations, marker=".", label="measurements")
# Add labels etc.
ax.set_xlabel("t")
ax.set_title(r"$x$")

# These two lines just remove duplicate labels (caused by the samples) from the legend
handles, labels = ax.get_legend_handles_labels()
by_label = dict(zip(labels, handles))

ax.legend(
    by_label.values(), by_label.keys(), loc="center left", bbox_to_anchor=(1, 0.5)
)

state_fig.tight_layout()
```


```python
from IPython.display import Image
Image('../../Python_probability_statistics_machine_learning_2E.png',width=200)
```

We have referred to regularization in the section [ch:ml:sec:logreg](#ch:ml:sec:logreg), but we want to develop this important
idea more fully. Regularization is the mechanism by which we
navigate the bias/variance trade-off.  To get started, let's
consider a classic constrained least squares problem,

$$
\begin{aligned}
& \underset{\mathbf{x}}{\text{minimize}}
& & \Vert\mathbf{x}\Vert_2^2 \\
& \text{subject to:}
& & x_0 + 2 x_1 = 1
\end{aligned}
$$

 where $\Vert\mathbf{x}\Vert_2=\sqrt{x_0^2+x_1^2}$ is the
$L_2$ norm.  Without the constraint, it would be easy to minimize
the objective function --- just take $\mathbf{x}=0$. Otherwise,
suppose we somehow know  that $\Vert\mathbf{x}\Vert_2<c$, then
the locus of points defined by this inequality is the circle in
[Figure](#fig:regularization_001). The constraint is the line in
the same figure. Because every value of $c$ defines a circle, the
constraint is satisfied  when the circle touches the line. The
circle can touch the line at many different points, but we are
only interested in the smallest such circle because this is a
minimization problem.  Intuitively, this means that we *inflate* a
$L_2$  ball at the origin and stop when it just touches the
contraint.  The point of contact is our $L_2$ minimization
solution.

The solution of the constrained $L_2$ minimization problem is at the point where the constraint (dark line) intersects the $L_2$ ball (gray circle) centered at the origin. The point of intersection is indicated by the dark circle. The two neighboring squares indicate points on the line that are close to the solution.

We can obtain the same result using the method of Lagrange
multipliers. We can rewrite the entire $L_2$ minimization problem
as one objective function using the Lagrange multiplier,
$\lambda$,

$$
J(x_0,x_1,\lambda) = x_0^2+x_1^2 + \lambda (1-x_0-x_1)
$$

 and solve this as an ordinary function using calculus. Let's
do this using Sympy.

```python
import sympy as S
S.var('x:2 l',real=True)
J=S.Matrix([x0,x1]).norm()**2 + l*(1-x0-2*x1)
sol=S.solve(map(J.diff,[x0,x1,l]))
print(sol)
```

    {l: 2/5, x0: 1/5, x1: 2/5}

**Programming Tip.**

Using  the `Matrix` object is overkill for this problem but it
does demonstrate how Sympy's matrix machinery works. In this case,
we are using the `norm` method to compute the $L_2$  norm of the
given elements. Using `S.var` defines Sympy variables and injects
them into the global namespace. It is more Pythonic to do
something like `x0 = S.symbols('x0',real=True)` instead but the
other way is quicker, especially for variables with many
dimensions.

 The solution defines the exact point where the line is
tangent to the circle in [Figure](#fig:regularization_001).  The
Lagrange multiplier has incorporated the constraint into the objective
function.

```python
%matplotlib inline

import numpy as np
from numpy import pi, linspace, sqrt
from matplotlib.patches import Circle
from matplotlib.pylab import subplots

x1 = linspace(-1,1,10)
dx=linspace(.7,1.3,3)
fline = lambda x:(1-x)/2.

fig,ax=subplots()
_=ax.plot(dx*1/5,fline(dx*1/5),'s',ms=10,color='gray')
_=ax.plot(x1,fline(x1),color='gray',lw=3)
_=ax.add_patch(Circle((0,0),1/sqrt(5),alpha=0.3,color='gray'))
_=ax.plot(1/5,2/5,'o',color='k',ms=15)
_=ax.set_xlabel('$x_1$',fontsize=24)
_=ax.set_ylabel('$x_2$',fontsize=24)
_=ax.axis((-0.6,0.6,-0.6,0.6))
ax.set_aspect(1)

fig.tight_layout()
fig.savefig('fig-machine_learning/regularization_001.png')
```

There is something subtle and very important about the nature of the solution,
however. Notice that there are other points very close to the solution on the
circle, indicated by the squares in [Figure](#fig:regularization_001). This
closeness could be a good thing, in case it helps us actually find a solution
in the first place, but it may be unhelpful in so far as it creates ambiguity.
Let's hold that thought and try the same problem using the $L_1$ norm instead
of the $L_2$ norm. Recall that

$$
\Vert \mathbf{x}\Vert_1 = \sum_{i=1}^d \vert x_i \vert
$$

 where $d$ is the dimension of the vector $\mathbf{x}$. Thus, we can
reformulate the same problem in the $L_1$  norm as in the following,

$$
\begin{aligned}
& \underset{\mathbf{x}}{\text{minimize}}
& & \Vert\mathbf{x}\Vert_1 \\
& \text{subject to:}
& & x_1 + 2 x_2 = 1
\end{aligned}
$$

  It turns out that this problem is somewhat harder to
solve using Sympy, but we have convex optimization modules in Python
that can help.

```python
from cvxpy import Variable, Problem, Minimize, norm1, norm
x=Variable((2,1),name='x')
constr=[np.matrix([[1,2]])*x==1]
obj=Minimize(norm1(x))
p= Problem(obj,constr)
p.solve()
print(x.value)
```

    [[1.31394645e-29]
     [5.00000000e-01]]

**Programming Tip.**

The `cvxy` module provides a unified and accessible interface to the powerful
`cvxopt` convex optimization package, as well as other open-source solver
packages.

 As shown in [Figure](#fig:regularization_002), the constant-norm
contour in the $L_1$ norm is shaped like a diamond instead of a circle.
Furthermore, the solutions found in each case are different.  Geometrically,
this is because inflating the circular $L_2$ reaches out in all directions
whereas the $L_1$ ball creeps out along the principal axes.  This effect is
much more pronounced in higher dimensional spaces where $L_1$-balls get more
spikey  [^spikey].  Like the $L_2$ case, there are also neighboring points on
the constraint line, but notice that these are not close to the boundary of the
corresponding $L_1$ ball, as they were in the $L_2$ case.  This means that
these would be harder to confuse with the optimal solution because they
correspond to a substantially different $L_1$ ball.

[^spikey]: We discussed the geometry of high dimensional space
when we covered the curse of dimensionality in the
statistics chapter.

To double-check our earlier $L_2$ result, we can also use the
`cvxpy` module to find the $L_2$ solution as in the following
code,

```python
constr=[np.matrix([[1,2]])*x==1]
obj=Minimize(norm(x,2)) #L2 norm
p= Problem(obj,constr)
p.solve()
print(x.value)
```

    [[0.2]
     [0.4]]

 The only change to the code is the $L_2$ norm and we get
the same solution as before.

Let's see what happens in higher dimensions for both $L_2$ and
$L_1$ as we move from two dimensions to four dimensions.

```python
x=Variable((4,1),name='x')
constr=[np.matrix([[1,2,3,4]])*x==1]
obj=Minimize(norm1(x))
p= Problem(obj,constr)
p.solve()
print(x.value)
```

    [[-3.64540020e-29]
     [-7.29271858e-29]
     [-8.32902339e-23]
     [ 2.50000000e-01]]

 And also in the $L_2$ case with the following code,

```python
constr=[np.matrix([[1,2,3,4]])*x==1]
obj=Minimize(norm(x,2))
p= Problem(obj,constr)
p.solve()
print(x.value)
```

    [[0.03333333]
     [0.06666667]
     [0.1       ]
     [0.13333333]]

 Note that the $L_1$ solution has selected out only one
dimension for the solution, as the other components are
effectively zero. This is not so with the $L_2$ solution, which
has meaningful elements in multiple coordinates.  This is because
the $L_1$ problem has many pointy corners in the four dimensional
space that poke at the hyperplane that is defined by the
constraint. This essentially means the subsets (namely, the points
at the corners) are found as solutions because these touch the
hyperplane. This effect becomes more pronounced in higher
dimensions, which is the main benefit of using the $L_1$ norm
as we will see in the next section.

```python
from matplotlib.patches import Rectangle, RegularPolygon

r=RegularPolygon((0,0),4,1/2,pi/2,alpha=0.5,color='gray')
fig,ax=subplots()
dx = np.array([-0.1,0.1])
_=ax.plot(dx,fline(dx),'s',ms=10,color='gray')
_=ax.plot(x1,fline(x1),color='gray',lw=3)
_=ax.plot(0,1/2,'o',color='k',ms=15)
_=ax.add_patch(r)
_=ax.set_xlabel('$x_1$',fontsize=24)
_=ax.set_ylabel('$x_2$',fontsize=24)
_=ax.axis((-0.6,0.6,-0.6,0.6))
_=ax.set_aspect(1)
fig.tight_layout()
fig.savefig('fig-machine_learning/regularization_002.png')
```

The diamond is the $L_1$ ball in two dimensions and the line is the constraint. The point of intersection is the solution to the optimization problem. Note that for $L_1$ optimization, the two nearby points on the constraint (squares) do not touch the $L_1$ ball. Compare this with [Figure](#fig:regularization_001).

## Ridge Regression

Now that we have a sense of the geometry of the situation, let's revisit
our classic linear regression probem. To recap, we want to solve the following
problem,

$$
\min_{\boldsymbol{\beta}\in \mathbb{R}^p} \Vert y - \mathbf{X}\boldsymbol{\beta}\Vert
$$

 where $\mathbf{X}=\left[
\mathbf{x}_1,\mathbf{x}_2,\ldots,\mathbf{x}_p \right]$ and $\mathbf{x}_i\in
\mathbb{R}^n$. Furthermore, we assume that the $p$ column vectors are linearly
independent (i.e., $\texttt{rank}(\mathbf{X})=p$). Linear regression produces
the $\boldsymbol{\beta}$ that minimizes the mean squared error above.  In the
case where $p=n$, there is a unique solution to this problem. However, when
$p<n$, then there are infinitely many solutions.

To make this concrete, let's work this out using Sympy. First,
let's define an example $\mathbf{X}$ and $\mathbf{y}$ matrix,

```python
import sympy as S
from sympy import Matrix
X = Matrix([[1,2,3],
            [3,4,5]])
y = Matrix([[1,2]]).T
```

  Now, we can define our coefficient vector $\boldsymbol{\beta}$
using the following code,

```python
b0,b1,b2=S.symbols('b:3',real=True)
beta = Matrix([[b0,b1,b2]]).T # transpose
```

 Next, we define the objective function we are trying to minimize

```python
obj=(X*beta -y).norm(ord=2)**2
```

**Programming Tip.**

The Sympy `Matrix` class has useful methods like the `norm` function
used above to define the objective function. The `ord=2` means we want
to use the $L_2$ norm. The expression in parenthesis evaluates to a
`Matrix` object.

 Note that it is helpful to define real variables using
the keyword argument whenever applicable because it relieves
Sympy's internal machinery of dealing with complex numbers.
Finally, we can use calculus to solve this by setting the
derivatives of the objective function to zero.

```python
sol=S.solve([obj.diff(i) for i in beta])
beta.subs(sol)
```

$\displaystyle \left[\begin{matrix}b_{2}\\\frac{1}{2} - 2 b_{2}\\b_{2}\end{matrix}\right]$

 Notice that the solution does not uniquely specify all the components
of the `beta` variable. This is a consequence of the $p<n$ nature of this
problem where $p=2$ and $n=3$.  While the the existence of this ambiguity does
not alter the solution,

```python
obj.subs(sol)
```

$\displaystyle 0$

 But it does change the length of the solution vector
`beta`,

```python
beta.subs(sol).norm(2)
```

$\displaystyle \sqrt{2 b_{2}^{2} + \left(2 b_{2} - \frac{1}{2}\right)^{2}}$

 If we want to minimize this length we can easily
use the same calculus as before,

```python
S.solve((beta.subs(sol).norm()**2).diff())
```

    [1/6]

 This provides the solution of minimum length
in the $L_2$ sense,

```python
betaL2=beta.subs(sol).subs(b2,S.Rational(1,6))
betaL2
```

$\displaystyle \left[\begin{matrix}\frac{1}{6}\\\frac{1}{6}\\\frac{1}{6}\end{matrix}\right]$

 But what is so special about solutions of minimum length? For machine
learning, driving the objective function to zero is symptomatic of overfitting
the data. Usually, at the zero bound, the machine learning method has
essentially memorized the training data, which is bad for generalization. Thus,
we can effectively stall this problem by defining a region for the solution
that is away from the zero-bound.

$$
\begin{aligned}
& \underset{\boldsymbol{\beta}}{\text{minimize}}
& & \Vert y - \mathbf{X}\boldsymbol{\beta}\Vert_2^2 \\
& \text{subject to:}
& & \Vert\boldsymbol{\beta}\Vert_2 < c
\end{aligned}
$$

 where $c$ is the tuning parameter. Using the same process as before,
we can re-write this as the following,

$$
\min_{\boldsymbol{\beta}\in\mathbb{R}^p}\Vert y-\mathbf{X}\boldsymbol{\beta}\Vert_2^2 +\alpha\Vert\boldsymbol{\beta}\Vert_2^2
$$

 where $\alpha$ is the tuning parameter. These are the *penalized* or
Lagrange forms of these problems derived from the constrained versions. The
objective function is penalized by the $\Vert\boldsymbol{\beta}\Vert_2$ term.
For $L_2$ penalization, this is called *ridge* regression.    This is
implemented in Scikit-learn as `Ridge`. The following code sets this up for
our example,

```python
from sklearn.linear_model import Ridge
clf = Ridge(alpha=100.0,fit_intercept=False)
clf.fit(np.array(X).astype(float),np.array(y).astype(float))
```

    Ridge(alpha=100.0, copy_X=True, fit_intercept=False, max_iter=None,
          normalize=False, random_state=None, solver='auto', tol=0.001)

 Note that the `alpha` scales of the penalty for the
$\Vert\boldsymbol{\beta}\Vert_2$. We set the `fit_intercept=False` argument to
omit the extra offset term from our example. The corresponding solution is the
following,

```python
print(clf.coef_)
```

    [[0.0428641  0.06113005 0.07939601]]

 To double-check the solution, we can use some optimization tools from
Scipy and our previous Sympy analysis, as in the following,

```python
from scipy.optimize import minimize
f   = S.lambdify((b0,b1,b2),obj+beta.norm()**2*100.)
g   = lambda x:f(x[0],x[1],x[2])
out = minimize(g,[.1,.2,.3]) # initial guess
out.x
```

    array([0.0428641 , 0.06113005, 0.07939601])

**Programming Tip.**

We had to define the additional `g` function from the lambda function we
created from the Sympy expression in `f` because the `minimize` function
expects a single object vector as input instead of a three separate arguments.

 which produces the same answer as the `Ridge` object. To
better understand the meaning of this result, we can re-compute the
mean squared error solution to this problem in one step using matrix
algebra instead of calculus,

```python
betaLS=X.T*(X*X.T).inv()*y
betaLS
```

$\displaystyle \left[\begin{matrix}\frac{1}{6}\\\frac{1}{6}\\\frac{1}{6}\end{matrix}\right]$

 Notice that this solves the posited problem exactly,

```python
X*betaLS-y
```

$\displaystyle \left[\begin{matrix}0\\0\end{matrix}\right]$

 This means that the first term in the objective function
goes to zero,

$$
\Vert y-\mathbf{X}\boldsymbol{\beta}_{LS}\Vert=0
$$

 But, let's examine the $L_2$ length of this solution versus
the ridge regression solution,

```python
print(betaLS.norm().evalf(), np.linalg.norm(clf.coef_))
```

    0.288675134594813 0.10898596412575512

 Thus, the ridge regression solution is shorter in the $L_2$
sense, but the first term in the objective function is not zero for
ridge regression,

```python
print((y-X*clf.coef_.T).norm()**2)
```

    1.86870864136429

 Ridge regression solution trades fitting error
($\Vert y-\mathbf{X} \boldsymbol{\beta}\Vert_2$) for solution
length ($\Vert\boldsymbol{\beta}\Vert_2$).

Let's see this in action with a familiar example from
[ch:stats:sec:nnreg](#ch:stats:sec:nnreg).  Consider [Figure](#fig:regularization_003).
For this example, we created our usual chirp signal and attempted to
fit it with a high-dimensional polynomial, as we did in
the section [ch:ml:sec:cv](#ch:ml:sec:cv).  The lower panel is the same except with ridge
regression. The shaded gray area is the space between the true signal
and the approximant in both cases. The horizontal hash marks indicate
the subset of $x_i$ values that each regressor was trained on.
Thus, the training set represents a non-uniform sample of the
underlying chirp waveform.  The top panel shows the usual polynomial
regression. Note that the regressor fits the given points extremely
well, but fails at the endpoint. The ridge regressor misses many of
the points in the middle, as indicated by the gray area, but does not
overshoot at the ends as much as the plain polynomial regression. This
is the basic trade-off for ridge regression. The Jupyter
notebook corresponding to this section has the code for this graph, but the main steps
are shown in the following,

```python
# create chirp signal
xi = np.linspace(0,1,100)[:,None]
# sample chirp randomly
xin= np.sort(np.random.choice(xi.flatten(),20,replace=False))[:,None]
# create sampled waveform
y = np.cos(2*pi*(xin+xin**2))
# create full waveform for reference
yi = np.cos(2*pi*(xi+xi**2))

# create polynomial features
from sklearn.preprocessing import PolynomialFeatures
qfit = PolynomialFeatures(degree=8) # quadratic
Xq = qfit.fit_transform(xin)
# reformat input as polynomial
Xiq = qfit.fit_transform(xi)

from sklearn.linear_model import LinearRegression
lr=LinearRegression() # create linear model
lr.fit(Xq,y) # fit linear model

# create ridge regression model and fit
clf = Ridge(alpha=1e-9,fit_intercept=False)
clf.fit(Xq,y)
```

    Ridge(alpha=1e-09, copy_X=True, fit_intercept=False, max_iter=None,
          normalize=False, random_state=None, solver='auto', tol=0.001)

```python
from sklearn.preprocessing import PolynomialFeatures
from sklearn.linear_model import LinearRegression
import numpy as np
from numpy import cos, pi

np.random.seed(1234567)
xi = np.linspace(0,1,100)[:,None]
xin = np.linspace(0,1,20)[:,None]
xin= np.sort(np.random.choice(xi.flatten(),20,replace=False))[:,None]
f0 = 1 # init frequency
BW = 2
y = cos(2*pi*(f0*xin+(BW/2.0)*xin**2))
yi = cos(2*pi*(f0*xi+(BW/2.0)*xi**2))

qfit = PolynomialFeatures(degree=8) # quadratic
Xq = qfit.fit_transform(xin)
Xiq = qfit.fit_transform(xi)

lr=LinearRegression() # create linear model
_=lr.fit(Xq,y)

fig,axs=subplots(2,1,sharex=True,sharey=True)
fig.set_size_inches((6,6))
ax=axs[0]
_=ax.plot(xi,yi,label='true',ls='--',color='k')
_=ax.plot(xi,lr.predict(Xiq),label=r'$\beta_{LS}$',color='k')
_=ax.legend(loc=0)
_=ax.set_ylabel(r'$\hat{y}$  ',fontsize=22,rotation='horizontal')
_=ax.fill_between(xi.flatten(),yi.flatten(),lr.predict(Xiq).flatten(),color='gray',alpha=.3)
_=ax.set_title('Polynomial Regression of Chirp Signal')
_=ax.plot(xin, -1.5+np.array([0.01]*len(xin)), '|', color='k',mew=3)

clf = Ridge(alpha=1e-9,fit_intercept=False)
_=clf.fit(Xq,y)
ax=axs[1]
_=ax.plot(xi,yi,label=r'true',ls='--',color='k')
_=ax.plot(xi,clf.predict(Xiq),label=r'$\beta_{RR}$',color='k')
_=ax.legend(loc=(0.25,0.70))
_=ax.fill_between(xi.flatten(),yi.flatten(),clf.predict(Xiq).flatten(),color='gray',alpha=.3)
# add rug plot
_=ax.plot(xin, -1.5+np.array([0.01]*len(xin)), '|', color='k',mew=3)
_=ax.set_xlabel('$x$',fontsize=22)
_=ax.set_ylabel(r'$\hat{y}$  ',fontsize=22,rotation='horizontal')
_=ax.set_title('Ridge Regression of Chirp Signal')
fig.savefig('fig-machine_learning/regularization_003.png')
```

The top figure shows polynomial regression and the lower panel shows polynomial ridge regression. The ridge regression does not match as well throughout most of the domain, but it does not flare as violently at the ends.  This is because the ridge constraint holds the coefficient vector down at the expense of poorer performance along the middle of the domain.

## Lasso Regression

Lasso regression follows the same basic pattern as ridge regression,
except with the $L_1$ norm in the objective function.

$$
\min_{\boldsymbol{\beta}\in\mathbb{R}^p}\Vert y-\mathbf{X}\boldsymbol{\beta}\Vert^2 +\alpha\Vert\boldsymbol{\beta}\Vert_1
$$

 The interface in Scikit-learn is likewise the same.
The following is the same problem as before using lasso
instead of ridge regression,

```python
X = np.matrix([[1,2,3],
               [3,4,5]])
y = np.matrix([[1,2]]).T
from sklearn.linear_model import Lasso
lr = Lasso(alpha=1.0,fit_intercept=False)
_=lr.fit(X,y)
print(lr.coef_)
```

    [0.         0.         0.32352941]

 As before, we can use the optimization tools in Scipy to solve this
also,

```python
from scipy.optimize import fmin
obj = 1/4.*(X*beta-y).norm(2)**2 + beta.norm(1)*l
f = S.lambdify((b0,b1,b2),obj.subs(l,1.0))
g = lambda x:f(x[0],x[1],x[2])
fmin(g,[0.1,0.2,0.3])
```

    Optimization terminated successfully.
             Current function value: 0.360297
             Iterations: 121
             Function evaluations: 221

    array([2.27469304e-06, 4.02831864e-06, 3.23134859e-01])

**Programming Tip.**

The `fmin` function from Scipy's optimization module uses an
algorithm that does not depend upon derivatives. This is useful
because, unlike the $L_2$ norm, the $L_1$ norm has sharp corners
that make it harder to estimate derivatives.

 This result matches the previous one from the
Scikit-learn `Lasso` object. Solving it using Scipy is motivating
and provides a good sanity check, but specialized algorithms are
required in practice. The following code block re-runs the lasso
with varying $\alpha$ and plots the coefficients in [Figure](#fig:regularization_004). Notice that as $\alpha$ increases, all
but one of the coefficients is driven to zero. Increasing $\alpha$
makes the trade-off between fitting the data in the $L_1$ sense
and wanting to reduce the number of nonzero coefficients
(equivalently, the number of features used) in the model. For a
given problem, it may be more practical to focus on reducing the
number of features in the model (i.e., large $\alpha$) than the
quality of the data fit in the training data. The lasso provides a
clean way to navigate this trade-off.

The following code loops over a set of $\alpha$ values and
collects the corresponding lasso coefficients to be plotted
in [Figure](#fig:regularization_004)

```python
o=[]
alphas= np.logspace(-3,0,10)
for a in alphas:
    clf = Lasso(alpha=a,fit_intercept=False)
    _=clf.fit(X,y)
    o.append(clf.coef_)
```

```python
fig,ax=subplots()
fig.set_size_inches((8,5))
k=np.vstack(o)
ls = ['-','--',':','-.']
for i in range(k.shape[1]):
    _=ax.semilogx(alphas,k[:,i],'o-',
                label='coef %d'%(i),
                color='k',ls=ls[i],
                alpha=.8,)

_=ax.axis(ymin=-1e-1)
_=ax.legend(loc=0)
_=ax.set_xlabel(r'$\alpha$',fontsize=20)
_=ax.set_ylabel(r'Lasso coefficients',fontsize=16)
fig.tight_layout()
fig.savefig('fig-machine_learning/regularization_004.png')
```

As $\alpha$ increases, more of the model coefficients are driven to zero for lasso regression.


# Use PY in Linear Algebrea

## Chapter Zero 乘法和庫

### 矩陣和向量的乘法
1. 點乘（dot product）：也被稱為「數量積」「內積」「點積」。結果的絕對值是一個向量在另一個向量方向上的投影的長度誠意另一個向量的長度，是一個標量。
2. 叉乘（cross product）：也被稱為「向量積」「外積」。對於兩個 $n$ 緯的向量叉乘，結果是一個和 $n-1$ 個向量都垂直的向量，方向符合右手定則。
3. 矩陣乘法：代表了線性變換的過程。

### python 的 sympy 库 和 numpy 库
说明：sympy 用於模擬和理解，numpy 用於实践。

#### sympy
`sympy`使用`*`代表矩陣乘法；

`sympy`使用`dot`表示點乘；

`sympy`使用`cross`表示叉乘

`det()`表示行列式；`inv()`表示擬矩陣；`adjugate()`伴隨矩陣

#### numpy
`numpy`使用`*`表示`array`乘法默認是按照位置的乘法，矩陣`matrix`默認的是矩陣乘法，混合起來的話默認是矩陣乘法；

`numpy`使用`dot`表示矩陣`matrix`和`array`的矩陣乘法；

`numpy`使用`cross`表示叉乘；

`numpy`使用`det()`表示行列式；

`.T` 返回自身的轉置；

`.H` 返回自身的共軛轉置；

`.J` 返回自身的逆矩陣；

`.A` 返回自身數據的2維數組的一個視圖（沒有做任何的拷貝）

## Chapter One Matrix

數學上，一個$m\times n$的矩陣是由一個$m$行(row)$n$列(column)元素排列成的矩形陣列，矩陣裡的元素可以是數字、符號或者數學式。在圖像處理、人工智能燈領域，使用矩陣來表示和處理數據非常常見。例：

$\mathbf{A}_{2 \times 3}=\begin{bmatrix} 5 & 2 & 7 \\ 1 & 3 & 4  \end{bmatrix}$

其中，矩陣$A$的下標$2\times 3$表示$A$是一個2行3列的矩陣。類似的，另一個示例：

$\mathbf{ B }_{ 4 \times 4 }=\begin{bmatrix} 5 & 2 & 7 & 6 \\ 1 & 3 & 4 & 2 \\ 7 & -1 & 9 & 0 \\ 8  & 2 & -2 & 3 \end{bmatrix}$

特別的，如果我們要表示矩陣$A$的第二行第二個元素：3，可以使用$A[2,2]$或者$a_{2,2}$。

在`python`中的`numpy`庫中提供了`ndarrary`類用於儲存高維數組及普通的數組運算，另外提供了`matrix`類用來支持矩陣運算（矩陣乘法）。下面來舉個例子：

```python
#first method to store the matrix
import numpy as np

a = np.matrix('5 2 7;1 3 4')
b = np.matrix('15 2 7 6;1 3 4 2;7 -1 9 0;8 2 -2 3')
print(a)#print matrix a
print(b)#print matrix b
```

    [[5 2 7]
     [1 3 4]]
    [[15  2  7  6]
     [ 1  3  4  2]
     [ 7 -1  9  0]
     [ 8  2 -2  3]]

```python
#second method to store the matrix
import numpy as np

a = np.matrix([[5,2,7],[1,3,4]])
b = np.matrix([[5,2,7,6],[1,3,4,2],[8,2,-2,3]])
print(a)#print matrix a
print(b)#print matrix b
```

    [[5 2 7]
     [1 3 4]]
    [[ 5  2  7  6]
     [ 1  3  4  2]
     [ 8  2 -2  3]]

這兩種形式完全等效，但是第一種更加直觀，不容易犯錯，所以推薦第一種方式（也很類似`LaTeX`中構成矩陣的寫法）；

要把一個`matrix`的對象轉換為`ndarray`對象，可以直接用`getA()`的方法。如果吧`ndarray`對象轉換成`matrx`對象可以用`asmatrix()`方法。

```python
b = a.getA()#turn a into ndarray
print(b)
print(type(b))# the type of b
c = np.asmatrix(b)
print(c)
print(type(c))# the type of c
```

    [[5 2 7]
     [1 3 4]]
    <class 'numpy.ndarray'>
    [[5 2 7]
     [1 3 4]]
    <class 'numpy.matrixlib.defmatrix.matrix'>

如果你要取出矩陣中的某個值，可以使用類似於數組的下標運算符。但是要注意，計算機的計數方式是從0開始的。例如，要取出$A[2,2]$，應該使用：

```python
a[1,1]
```

    3

### 運算

#### 加
矩陣加法的定義非常符合直覺。假設有$\mathbf{ A }_{ 3 \times 3 }=\begin{bmatrix} 1 & 0 & 1 \\ 1 & 2 & 1 \\ 2 & 1 & 1 \end{bmatrix}$，$\mathbf{ B }_{ 3 \times 3 }=\begin{bmatrix} 2 & 1 & -1 \\ 0 & -1 & 2 \\ 2 & -1 & 0 \end{bmatrix}$，
 則：
$\mathbf{A}+\mathbf{B} = \begin{bmatrix} 1 & 0 & 1 \\ 1 & 2 & 1 \\ 2 & 1 & 1 \end{bmatrix} + \begin{bmatrix} 2 & 1 & -1 \\ 0 & -1 & 2 \\ 2 & -1 & 0 \end{bmatrix} = \begin{bmatrix} 1+2 & 0+1 & 1+(-1) \\ 1+ 0 & 2+(-1) & 1+2 \\ 2+2 & 1+(-1) & 1+0 \end{bmatrix} = \begin{bmatrix} 3 & 1 & 0 \\ 1 & 1 & 3 \\ 4 & 0 & 1 \end{bmatrix}$

要注意，兩個矩陣的行數和列數必須相同，否則無定義。下面是`python`示例：

```python
a = np.matrix('1 0 1;1 2 1;2 1 1')
b = np.matrix('2 1 -1;0 -1 2;2 -1 0')
print(a+b)# print a+b
```

    [[3 1 0]
     [1 1 3]
     [4 0 1]]

很容易看出，矩陣的加法滿足交換律和結合律，即$A+B=B+A$，$(A+B)+C=A+(B+C)$。

#### 減
矩陣減法也和加法一樣簡單。對於上面給出的$A$和$B$，有：

$\mathbf{A}-\mathbf{B}=\begin{bmatrix} 1 & 0 & 1 \\ 1 & 2 & 1 \\ 2 & 1 & 1 \end{bmatrix}-\begin{bmatrix} 2 & 1 & -1 \\ 0 & -1 & 2 \\ 2 & -1 & 0 \end{bmatrix}=\begin{bmatrix} 1-2 & 0-1 & 1-(-1) \\ 1-0 & 2-(-1) & 1-2 \\ 2-2 & 1-(-1) & 1-0 \end{bmatrix}=\begin{bmatrix} -1 & -1 & 2 \\ 1 & 3 & -1 \\ 0 & 2 & 1 \end{bmatrix}$

同樣，相減的兩個矩陣行數和列數必須完全相同，否則無定義。下面是`python`示例：

```python
print(a-b)
```

    [[-1 -1  2]
     [ 1  3 -1]
     [ 0  2  1]]

#### 乘
矩陣乘法的定義是$A_{i\times j}$矩陣的每一行的元素分別與$B_{j\times k}$矩陣的每一列的元素兩兩相乘並詳加，從而，得到新的矩陣$C_{j\times k$。兩個矩陣能相乘的充份必要條件是第一個矩陣的列數等於第二個矩陣的行數相等，否則，無定義。$\begin {aligned}
\mathbf{A} \times \mathbf{B} &=\begin{bmatrix} 1 & 0 & 1 \\ 1 & 2 & 1 \\ 2 & 1 & 1 \end{bmatrix}\times \begin{bmatrix} 2 & 1 & -1 \\ 0 & -1 & 2 \\ 2 & -1 & 0 \end{bmatrix} \\\ &=\begin{bmatrix} 1\cdot 2+0\cdot 0+1\cdot 2 & 1\cdot 1+0\cdot (-1)+1\cdot (-1) & 1\cdot (-1)+0\cdot 2+1\cdot 0 \\ 1\cdot 2+2\cdot 0+1\cdot 2 & 1\cdot 1+2\cdot (-1)+1\cdot (-1) & 1\cdot (-1)+2\cdot 2+1\cdot 0 \\ 2\cdot 2+1\cdot 0+1\cdot 2 & 2\cdot 1+1\cdot (-1)+1\cdot (-1) & 2\cdot (-1)+1\cdot 2+1\cdot 0 \end{bmatrix}\\\ &=\begin{bmatrix} 4 & 0 & -1 \\ 4 & -2 & 3 \\ 6 & 0 & 0 \end{bmatrix}
\end {aligned}$

再舉一個行列數不同的例子，假設有$\mathbf{C}_{2 \times 3} = \begin{bmatrix} 5 & 7 & 2 \\ 4 & 3 & 1 \end{bmatrix}$和$\mathbf{D}_{3 \times 1} = \begin{bmatrix} 1 \\ 5 \\ 6 \end{bmatrix}$，那麼，我們可以得出：$\mathbf{C}\times \mathbf{D} = \begin{bmatrix} 5 & 7 & 2 \\ 4 & 3 & 1 \end{bmatrix}\times \begin{bmatrix} 1 \\ 5 \\ 6 \end{bmatrix}
 =\begin{bmatrix} 5 \cdot 1+ 7 \cdot 5+ 2\cdot 6 \\ 4\cdot 1+3\cdot 5+1\cdot 6  \end{bmatrix} =\begin{bmatrix} 52 \\ 25 \end{bmatrix}$

與初等代數中學的乘法不同，矩陣的乘法並不滿足交換律，即，$A \times B \neq B \times A$，但是，它滿足分配律：$(A\times B)\times C = A\times (B\times C)$。

再介紹兩個特殊的矩陣：

1. 單位矩陣（Identity Matrix）他的特點是與行數和列數相等，對角線上的值都是1，其他值都是0。另一個特點是：它與任意矩陣$A$相乘結果等於$A$。

$\mathbf{I}_{3 \times 3} = \begin{bmatrix} 1 & 0 & 0 \\ 0 & 1 & 0 \\ 0 & 0 & 1 \end{bmatrix}$這是一個 $3\times 3$的單位矩陣

2. 零矩陣（Null Matrix）顧名思義就是全部元素都是0的矩陣。零矩陣乘以任何矩陣都是零矩陣，與任何矩陣$A$都等於$A$。

```python
#python示例
print("this is a times b:")
print(a*b)
print("this is b times a:")
print(b*a)
```

    this is a times b:
    [[ 4  0 -1]
     [ 4 -2  3]
     [ 6  0  0]]
    this is b times a:
    [[ 1  1  2]
     [ 3  0  1]
     [ 1 -2  1]]

```python
c = np.matrix('5 7 2;4 3 1')
d = np.matrix('1;5;6')
print("this is c times d:")
print(c*d)
print("this is a times b times d:")
print(a*b*d)
print("first,we user b times d, then we use a to times the b*d:")
print(a*(b*d))
I = np.eye(3)
#we defined I as a 3 by 3 indentity matrix;eye() equals to'I = np.matrix('1 0 0;0 1 0;0 0 1')'
print("This is a times I:")
print(a*I)
print("This is I times a:")
print(I*a)
z = np.zeros([3,3])
#we define z as a 3 by 3 null matrix; np.zeros() equals to'z = np.matrix('0 0 0;0 0 0;0 0 0')'
print("This is a times z:")
print(a*z)
print("This is z times a:")
print(z*a)
```

    this is c times d:
    [[52]
     [25]]
    this is a times b times d:
    [[-2]
     [12]
     [ 6]]
    first,we user b times d, then we use a to times the b*d:
    [[-2]
     [12]
     [ 6]]
    This is a times I:
    [[1. 0. 1.]
     [1. 2. 1.]
     [2. 1. 1.]]
    This is I times a:
    [[1. 0. 1.]
     [1. 2. 1.]
     [2. 1. 1.]]
    This is a times z:
    [[0. 0. 0.]
     [0. 0. 0.]
     [0. 0. 0.]]
    This is z times a:
    [[0. 0. 0.]
     [0. 0. 0.]
     [0. 0. 0.]]

#### ~~除~~（求逆）

矩陣中沒有除法這種操作，但是有一個相似的運算，叫做求逆。在線性代數中，給定一個$n$階的方陣$A$ ，若存在一個$n$階矩陣$B$，使得$AB = BA = I_n$，其中$I_n$為$n$階的單位矩陣，則稱$A$是**可逆**的，而$B$則是$A$的逆矩陣，記作：$A^{-1}$。

##### 伴隨陣法：
$\mathbf{A}^{-1}=\frac{1}{|\mathbf{A}|}\begin{bmatrix}A_{11} & A_{21} & \ldots & A_{n1} \\ A_{12} & A_{22} & \ldots & A_{n2} \\ \ldots & \ldots & \ldots & \ldots \\ A_{1n} & A_{2n} & \ldots & A_{nn} \end{bmatrix}$，其中$A_{ij}$是$|A|$中元素$a_{ij}$的代數余子式。

矩陣$\begin{bmatrix}A_{11} & A_{21} & \ldots & A_{n1} \\ A_{12} & A_{22} & \ldots & A_{n2} \\ \ldots & \ldots & \ldots & \ldots \\ A_{1n} & A_{2n} & \ldots & A_{nn} \end{bmatrix}$稱為矩陣$A$ 的伴隨矩陣，記作$A*$

##### 初等變換法：
如果矩陣$A$和$B$互逆，則$AB=BA=I$。由$AB=BA$可以得出：兩個矩陣都是方陣；由$AB=I$以及定理“兩個矩陣的乘積的行列式等於兩個矩陣的行列式的乘積”可得：兩個矩陣的行列式不等於零。現在假設三階矩陣${ \mathbf{A} }_{ 3 \times 3 }=\begin{bmatrix} 1 & 0 & 1 \\ 1 & 2 & 1 \\ 2 & 1 & 1 \end{bmatrix}$，則：

$\begin{aligned}
\begin{bmatrix}\mathbf{A} \mathbf{I}\end{bmatrix}
& \rightarrow
\begin{bmatrix}
1 & 0 & 1 & 1 & 0 & 0 \\
1 & 2 & 1 & 0 & 1 & 0 \\
2 & 1 & 1 & 0 & 0 & 1
\end{bmatrix}
\rightarrow
\begin{bmatrix}
1 & 0 & 1 & 1 & 0 & 0 \\
0 & 2 & 0 & -1 & 1 & 0 \\
2 & 1 & 1 & 0 & 0 & 1
\end{bmatrix}
\rightarrow
\begin{bmatrix}
1 & 0 & 1 & 1 & 0 & 0 \\
0 & 1 & 0 & -0.5 & 0.5 & 0 \\
2 & 1 & 1 & 0 & 0 & 1
\end{bmatrix}\\
& \rightarrow
\begin{bmatrix}
1 & 0 & 1 & 1 & 0 & 0 \\
0 & 1 & 0 & -0.5 & 0.5 & 0 \\
1 & 1 & 0 & -1 & 0 & 1
\end{bmatrix}
\rightarrow
\begin{bmatrix}
1 & 0 & 1 & 1 & 0 & 0 \\
0 & 1 & 0 & -0.5 & 0.5 & 0 \\
1 & 0 & 0 & -0.5 & -0.5 & 1
\end{bmatrix}
\rightarrow
\begin{bmatrix}
0 & 0 & 1 & 1.5 & 0.5 & -1 \\
0 & 1 & 0 & 0 & 0.5 & 0 \\
1 & 0 & 0 & -0.5 & -0.5 & 1
\end{bmatrix}\\
&\rightarrow
\begin{bmatrix}
1 & 0 & 0 & -0.5 & -0.5 & 1 \\
0 & 1 & 0 & -0.5 & 0.5 & 0 \\
0 & 0 & 1 & 1.5 & 0.5 & -1
\end{bmatrix}
\end{aligned}$

所以：$\mathbf{A}^{-1}=\begin{bmatrix}-0.5 & -0.5 & 1 \\ -0.5 & 0.5 & 0 \\ 1.5 & 0.5 & -1\end{bmatrix}$

### 奇異矩陣（Singular Matrix）
如果一個方陣$A$滿足條件$|A|(det(A))\neq 0$，則稱$A$為非奇異矩陣（non-singular matrix），否則則稱為奇異矩陣。簡單的說**沒有逆矩陣的矩陣就是奇異矩陣，有逆矩陣的矩陣就是非奇異矩陣**

`python`求逆示例：

```python
a = np.matrix('1 0 1; 1 2 1; 2 1 1')
print(a.I) #the inverse of matrix a
print(a*a.I) #a times a inverse equals to indentity matrix
f = np.matrix('0 1;0 0')
print(f.I)#because f has no inverse, f is a singular matrix
```

### 矩陣的轉置（Transport）
在線性代數中，矩陣$A$的轉置是另一個矩陣$A^T$，如下所示：

* 把$A$的橫行寫成$A^T$的縱列；
* 把$A$的縱列寫成$A^T$的橫行；
形式上說就是，$m\times n$的矩陣$A$的轉置是$n\times m$矩陣。

矩陣$\underset{m\times n}{\mathbf{A}} = \begin{bmatrix}a_{11} & a_{12} & \ldots & a_{1n} \\ a_{21} & a_{22} & \ldots & a_{2n} \\ \ldots \\ a_{m1} & a_{m2} & \ldots & a_{mn}\end{bmatrix}$的轉置定義為$\underset{n\times n}{A^{T}} = \begin{bmatrix}a_{11} & a_{21} & \ldots & a_{m1} \\ a_{12} & a_{22} & \ldots & a_{m2} \\ \ldots \\ a_{1n} & a_{2n} & \ldots & a_{mn}\end{bmatrix}$

```python
#python 示例：
a = np.matrix('2 4;1 3')
print(a.T)#the transport of a
b = np.matrix('1 2 3;4 5 6')
print(b.T)#the transport of b
```

**特別的，矩陣的轉置有一個性質：矩陣的轉置等於矩陣調換後分別做轉置的乘積：$（A\bullet B)^T = B^T\bullet A^T$**

```python
a = np.matrix('2 4;1 3')
b = np.matrix('1 6;2 5')
c = a*b
print(c.T)
print(b.T*a.T)
```

```python
import numpy as np
import matplotlib.pyplot as plt
from pylab import *

xlim(-3, 3)
ylim(0, 3.5)
plt.plot([0,2.5],[3.5,0],'blue',[-1,3],[0,3.5],'red')
```

可以用矩陣表示成：$\begin{bmatrix}
3 & 2 \\
-1 & 1
\end{bmatrix}
\begin{bmatrix}
x \\
y
\end{bmatrix}
=
\begin{bmatrix}
7\\
1
\end{bmatrix}$

設$\begin{bmatrix}3 & 2 \\-1 & 1\end{bmatrix}$為矩陣$A$，將等式兩邊左乘一個$A$的逆，就可以得到：

$\begin{aligned}
A^{-1}A
\begin{bmatrix}
x \\ y
\end{bmatrix}
&=
A^{-1}
\begin{bmatrix}
7\\
1
\end{bmatrix}\\\
&=
\frac{1}{|A|}\begin{bmatrix}1 & -2 \\ 1 & 3\end{bmatrix}
\begin{bmatrix}
7\\
1
\end{bmatrix}\\\
&=
\frac{1}{5}\begin{bmatrix}1 & -2 \\ 1 & 3\end{bmatrix}
\begin{bmatrix}
7\\
1
\end{bmatrix}\\\
&=
\frac{1}{5}\begin{bmatrix}5 \\ 10\end{bmatrix}
\end{aligned}$

因此，$\begin{bmatrix}x \\ y\end{bmatrix}=\begin{bmatrix}1 \\ 2\end{bmatrix}$

雖然這個方法在二元一次方程這看起來還沒有初中學的校園法解得快，但使用矩陣的好處在於對於更高維的數據，比如成白上千的未知數，這個解法依然有效。

在`python`中，我們可以使用`numpy`的線性代數算法庫`linalg`提供的`solve`方法來求解方程組。例：

```python
a = np.matrix('3 2; -1 1')
b = np.matrix('7; 1')
print(np.linalg.solve(a, b))
```

### 求向量組合
假設有向量$\vec { a }  =  \begin{bmatrix} 3 \\ -1 \end{bmatrix}$，$\vec { b }  =  \begin{bmatrix} 2 \\ 1 \end{bmatrix}$，求兩者如何組成新向量$\vec { c }  =  \begin{bmatrix} 7 \\ 1 \end{bmatrix}$？

```python
import numpy as np
import matplotlib.pyplot as plt

soa = np.array([[0, 0, 3, -1], [0, 0, 2, 1], [0, 0, 7, 1]])
X, Y, U, V = zip(*soa)
plt.figure()
ax = plt.gca()
ax.quiver(X, Y, U, V, angles='xy', scale_units='xy', scale=1)
ax.set_xlim([-1, 10])
ax.set_ylim([-1, 10])
plt.draw()
plt.show()
```

如果用$x$和$y$分別表示兩個向量的背書，這個問題就可以用矩陣表示成：$\begin{bmatrix}3 \\-1 \end{bmatrix}x + \begin{bmatrix}2 \\1\end{bmatrix}y=\begin{bmatrix}7\\1\end{bmatrix}$

這樣就把這個問題轉換成和上一個問題同構了，使用同樣的方法可以得出：$\begin{bmatrix}x \\ y\end{bmatrix}=\begin{bmatrix}1 \\ 2\end{bmatrix}$

```python
import numpy as np
import matplotlib.pyplot as plt

soa = np.array([[0, 0, 3, -1], [0, 0, 4, 2], [0, 0, 7, 1],[4,2,3,-1]])
X, Y, U, V = zip(*soa)
plt.figure()
ax = plt.gca()
ax.quiver(X, Y, U, V, angles='xy', scale_units='xy', scale=1)
ax.set_xlim([-1, 10])
ax.set_ylim([-1, 10])
plt.draw()
plt.show()
```

## Chapter Two  Determinant

### 什麼是行列式

**行列式**（Dterminant），在顯示代數中，行列式是一個值，一個數值可以告訴我們一個矩陣是否是奇異的。對於每一個$n\times n$的矩陣$A$，據對應一個標量$det(A)$，它的值將告訴我們矩陣是否為非奇異的。

1. $2\times 2$ 矩陣的行列式值

$det(\
\left[\begin{array}{cc}
    a & b\\
    c & d\\
\end{array}\right]
\  ) = ad - bc$

我們也可以用`sympy`進行核對：

```python
from sympy import *

a,b,c,d = symbols("a,b,c,d")
M1=Matrix([[a, b], [c, d]])
print(M1.det())# print the determinant of matrix M1
```

### 行列式的幾何意義：二維和三維歐式空間中的例子

行列式的一個自然的怨氣就是$n$維平行體的體積。行列式的定義和$n$維平行體的體積有著本質上的關聯。

在二維空間中，行列式如上面例子所示；比如說：$det(A,B) =\
\left|\begin{array}{cc}
    2 & 3\\
    1 & 4\\
\end{array}\right|
\ = 2\bullet 4 - 3\bullet 1 = 5$，我們經過計算可以知道，當係數是實數的時候，行列式表示的就是向量$\vec A$和$\vec B$ 形成的平行四邊形的**有向面積**

```python
import numpy as np
import matplotlib.pyplot as plt
from pylab import *

plt.plot([0,2],[0,1],'blue',[3,5],[4,5],'blue',[0,3],[0,4],'red',[2,5],[1,5],'red')
```

```python
import numpy as np

a = np.matrix([[2,1],[3,4]])
print(np.linalg.det(a))# print the determinant of matrix M1
```

我們通過計算，可以得到這個矩陣的行列式等於$5$，可以看出，上圖由紅線和藍線形成的是一個平行四邊形，我們可以通過計算得出，它的面積等於$5$

*這裡只是簡單介紹二維行列式，具體請查閱課本**

那如果是三位空間呢？在三維的有向空間中，三個三維向量的行列式是$\left|\begin{array}{ccc}
    x & x' & x''\\
    y & y' & z''\\
    z & z' & z''
\end{array}\right|
\ = xy'z'' + x'y''z + x''yz' - xy''z' - xyz'' - x''y'z $

比如說，三個向量$(2,1,5),(6,0,8)$ 和 $(3,2,4)$ 的行列式為：
$\left|\begin{array}{ccc}
    2 & 6 & 3\\
    1 & 0 & 2\\
    5 & 8 & 4
\end{array}\right| = 28
$

```python
import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D

p0 = [2, 1,5]
p1 = [6,0,8]
p2 = [3,2,4]

origin = [0,0,0]
X, Y, Z = zip([0,0,0],origin,origin)
U, V, W = zip(p0,p1,p2)

fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
ax.quiver(X,Y,Z,U,V,W,arrow_length_ratio=0.001)
ax.set_xlim([-1, 5])
ax.set_ylim([-1, 5])
ax.set_zlim([-1, 8])

plt.show()
```

類比一下二維向量的行列式，我們可以猜測一下三維向量組的行列式的幾何意義，就是這三個向量行程的平行六面體的**有向體積**
；我們可以用`python`來算它的行列式：

```python
a = np.matrix([[2,1,5],[6,0,8],[3,2,4]])
print(np.linalg.det(a))
```

***特別的，如果向量組中，所有向量共線或者共面（線性相關），面積或體積為零**，什麼是線性相關，我們會在後面解釋*

## Chapter Three Vector

### What is Vector

在初中（或高中）物理，我們就已經接觸過向量了，那向量到底是什麼？它是指一個同事具有大小和方向，且滿足平行四邊形法則的集合對象。一般地，同時滿足具有大小和方向兩個性質的集合對象就可以稱作為向量（物理中的電流：有方向，有大小，但是它不是向量，這是特殊情況，我們在線性代數中不討論）。與向量相對的叫做**標量**（scalar），一般標量和向量最大的區別就是是否有向。

在$R^n$空間中定義$\vec V$，可以用一個包含$n$個實數的有序集來表示：$\vec V = \begin{bmatrix} v_1 \\ v_2 \\ \dots \\ v_n\end{bmatrix}$，這個有序集中的每一個元素被稱為向量的分量（在某一維度的量）。例如，一個在$R^2$空間的向量$\begin{bmatrix} 2 \\ 1 \end{bmatrix}$,我們也可以用$(2,1)$或$<2,1>$來表示：

```python
import numpy as np
import matplotlib.pyplot as plt

soa = np.array([[0, 0, 2, 1]])
X, Y, U, V = zip(*soa)
plt.figure()
ax = plt.gca()
ax.quiver(X, Y, U, V, angles='xy', scale_units='xy', scale=1)
ax.set_xlim([0, 5])
ax.set_ylim([0, 5])
plt.draw()
plt.show()
```

向量的長度被定義為$\parallel v \parallel = \sqrt{v_1^2 + v_2^2 +\dots + v_n^2}$；當然的，當$\parallel v \parallel = 1$，我們稱這個向量是**單位向量**

### 基本運算

#### 加法

向量$a$和向量$b$的加法定義為：$\mathbf{a} + \mathbf{b} = \begin{bmatrix} a_1 + b_2 \\ a_2 + b_2 \\ \dots \\a_n + b_n \end{bmatrix}$

下面通過畫圖來示意向量$\mathbf{a} = \begin{bmatrix} -1 \\ 2 \end{bmatrix}$與$\mathbf{b} = \begin{bmatrix}3 \\ 1\end{bmatrix}$相加，等於$\begin{bmatrix} 2 \\ 3 \end{bmatrix}$

```python
import numpy as np
import matplotlib.pyplot as plt

soa = np.array([[0, 0, -1, 2], [0, 0, 3, 1], [0, 0, 2, 3],[3,1,-1,2]])
X, Y, U, V = zip(*soa)
plt.figure()
ax = plt.gca()
ax.quiver(X, Y, U, V, angles='xy', scale_units='xy', scale=1)
ax.set_xlim([-2, 5])
ax.set_ylim([-1, 5])
plt.draw()
plt.show()
```

可以看出，向量的加法是遵從平行四邊形法則的，在`python`中，我們可以直接用`numpy`的`ndarray`來表示向量：

```python
import numpy as np
a = np.array([-1,2])
b = np.array([3,1])
print(a+b)
```

#### 減法

了解了加法了，減法就相當於加了一個反方向的向量，$\mathbf{a} - \mathbf{b} = \begin{bmatrix} a_1 - b_1 \\ a_2 - b_2 \\ \ldots \\ a_n - b_n \end{bmatrix}$，下面通過畫圖來示意向量$\mathbf{a} = \begin{bmatrix} -1 \\ 2 \end{bmatrix}$與$\mathbf{b} = \begin{bmatrix}3 \\ 1\end{bmatrix}$相減，等於$\begin{bmatrix} -4 \\ 1 \end{bmatrix}$

```python
import numpy as np
import matplotlib.pyplot as plt

soa = np.array([[0, 0, -1, 2], [0, 0, -3, -1], [0, 0, -4, 1],[-3,-1,-1,2]])
X, Y, U, V = zip(*soa)
plt.figure()
ax = plt.gca()
ax.quiver(X, Y, U, V, angles='xy', scale_units='xy', scale=1)
ax.set_xlim([-5, 3])
ax.set_ylim([-3, 3])

plt.show()
```

```python
import numpy as np
a = np.array([-1, 2])
b = np.array([3, 1])
print(a - b)

```

#### 乘法

##### 標量乘以向量

標量$c$乘以向量$\mathbf{a}$定義為：$c \cdot \mathbf{a} = \begin{bmatrix} c \cdot a_1 \\ c \cdot a_2 \\ \ldots \\ c \cdot a_n \end{bmatrix} = \begin{bmatrix} a_1 \cdot c \\ a_2 \cdot c \\ \ldots \\ a_n \cdot c \end{bmatrix}$

用圖來示意$\mathbf{a} \begin{bmatrix} -1 \\2 \end{bmatrix}$乘以一個標量$3$得到$\begin{bmatrix}-3 \\ 6 \end{bmatrix}$

```python
import numpy as np
a = np.array([-1,2])
print(3 * a)
```

##### 向量點積

定義如下：$\vec{\mathbf{a}}\cdot \vec{\mathbf{b}} = \begin{bmatrix} a_1 \\ a_2 \\ \ldots \\ a_n\end{bmatrix} \cdot \begin{bmatrix} b_1 \\ b_2 \\ \ldots \\ b_n \end{bmatrix} = a_{1}b_{1} + a_{2}b_{2} + \ldots + a_{n}b_{n}$ 可以看出，向量點積得到的是一個標量；

例如: $\begin{bmatrix} 3 \\ 5 \\ 2 \end{bmatrix} \cdot \begin{bmatrix} 1 \\ 4 \\ 7 \end{bmatrix} = 3 \cdot 1 + 5 \cdot 4 + 2 \cdot 7 = 37$

```python
import numpy as np

vec_1 = np.array([3,5,2])
vec_2 = np.array([1,4,7])
print(np.dot(vec_1, vec_2)) # dot product of vectors
print(np.cross(vec_1, vec_2)) # cross multipication of vectors
```

```python
import numpy as np
import matplotlib.pyplot as plt

soa = np.array([[0, 0, -1, 2], [0, 0, -3, 6]])
X, Y, U, V = zip(*soa)
plt.figure()
ax = plt.gca()
ax.quiver(X, Y, U, V, angles='xy', scale_units='xy', scale=1)
ax.set_xlim([-3, 3])
ax.set_ylim([-3, 8])
plt.draw()
plt.show()
```

可以看出，第一個答案是點乘出來的結果，在`python-numpy`中，我們用`np.dot(vector, vector)`來計算兩個或多個向量的點積。而`(vector * vector)`則是計算向量的叉乘結果，下面來介紹一下叉乘（向量積）：

它是向量和向量的乘積，不過結果是一個向量，它的集合意義是所得的向量和被乘向量所在的平面垂直，方向由右手定則規定，大小是兩個被乘向量張成的平行四邊形的面積。所以，向量積不滿足交換律。下面主要介紹在$\mathbf{R}^2$和$\mathbf{R}^3$的情況下的向量積：

$\mathbf{R}^2$的向量積：

$\begin{bmatrix} a_1 \\ a_2\end{bmatrix} \times \begin{bmatrix} b_1 \\ b_2 \end{bmatrix} = \begin{bmatrix} a_1 \cdot b_2 - a_2 \cdot b_1\end{bmatrix}$

例如：$\begin{bmatrix} 1 \\ 2 \end{bmatrix} \times \begin{bmatrix} 3 \\4 \end{bmatrix}
=\begin{bmatrix} 1 \cdot 4 - 3 \cdot 2 \end{bmatrix}= \begin{bmatrix}-2\end{bmatrix}$

$\mathbf{R}^3$的向量積：

$\begin{bmatrix} a_1 \\ a_2 \\ a_3\end{bmatrix} \times \begin{bmatrix} b_1 \\ b_2 \\ b_3 \end{bmatrix} = \begin{bmatrix} a_2 \cdot b_3 - a_3 \cdot b_2 \\ a_3 \cdot b_1 - a_1 \cdot b_3 \\ a_1 \cdot b_2 - a_2 \cdot b_1\end{bmatrix}$

例如：$\begin{bmatrix} 3 \\ 5 \\ 2 \end{bmatrix} \times \begin{bmatrix} 1 \\ 4 \\ 7 \end{bmatrix} =\begin{bmatrix} 5 \cdot 7 - 2 \cdot 4 \\ 2 \cdot 1 - 3 \cdot 7 \\ 3 \cdot 4 - 5 \cdot 1\end{bmatrix}= \begin{bmatrix} 27 \\ -19 \\ 7\end{bmatrix}$

可以看出，向量積的結果就是一個新的向量。

```python
import numpy as np
a = np.array([3, 5, 2])
b = np.array([1, 4, 7])
print(np.cross(a, b)) # cross multipication of vec a and vec b
```

之前我們所講的向量長度的定義式：$\left\| \vec{\mathbf{v}} \right\| = \sqrt{v_{1}^{2} + v_{2}^{2} + \ldots + v_{n}^{2}}$，如果我們連立向量點積的定義就可以得出$\parallel \mathbf{v} \parallel = \sqrt{\vec{v}\cdot \vec{v}}$

關於點積，還有一個重要的重要的性質，就是**柯西不等式**：
**對於兩個非零向量，$\vec{x}, \vec{y} \in \mathbf{R}^n, |\vec{x}\cdot \vec{y}| \leq \parallel \mathbf{x}\parallel \parallel\mathbf{y} \parallel$，當且僅當$\vec{\mathbf{x}} = c\vec{\mathbf{y}}$時，等號成立**。證明如下：

$\begin{align}
\left\|\vec{\mathbf{x}} + \vec{\mathbf{y}}\right\|^2 & = (\vec{\mathbf{x}} + \vec{\mathbf{y}})\cdot (\vec{\mathbf{x}} + \vec{\mathbf{y}}) \\\
 & = \left\|\vec{\mathbf{x}}\right\|^2 + 2\vec{\mathbf{x}}\vec{\mathbf{y}} + \left\|\vec{\mathbf{y}}\right\|^2 \\\
& \le
\left\|\vec{\mathbf{x}}\right\|^2 + 2\left\|\vec{\mathbf{x}}\right\|\left\|\vec{\mathbf{y}}\right\| + \left\|\vec{\mathbf{y}}\right\|^2
\end{align}$

所以：

$\left\|\vec{\mathbf{x}} + \vec{\mathbf{y}}\right\|^2 \le (\left\|\vec{\mathbf{x}}\right\| + \left\|\vec{\mathbf{y}}\right\|)^2$

兩邊開平方後：

$\left\|\vec{\mathbf{x}} + \vec{\mathbf{y}}\right\| \le
\left\|\vec{\mathbf{x}}\right\| + \left\|\vec{\mathbf{y}}\right\|$

這也就是我們高中三角函數裡學過的**三角不等式**，從幾何的角度來說，向量的點積和向量的夾角$\theta$的餘弦值時有關的：$\vec{\mathbf{a}}\cdot\vec{\mathbf{b}} = \left\|\vec{\mathbf{a}}\right\|\left\|\vec{\mathbf{b}}\right\|cos\theta$

從另一個角度來看，這其實就是一個向量在另一個向量上的投影: $Proj_L({\vec{\mathbf{x}}}) =  c\vec{\mathbf{v}} = (\frac{\vec{\mathbf{x}}\cdot \vec{\mathbf{v}}}{\vec{\mathbf{v}}\cdot \vec{\mathbf{v}}})\vec{\mathbf{v}}$

```python
import numpy as np
#the first way to define the function
def get_projection(a, b):
    return a.dot(b)*1.0*b/b.dot(b)

a = np.array([1, 2])
b = np.array([2, 2])
print(get_projection(a, b))
```

```python
import numpy as np
#the second way to define the function
find_projection = lambda a, b : a.dot(b) * 1.0 * b/b.dot(b)

a = np.array([1, 2])
b = np.array([2, 2])

print(find_projection(a,b))
```

這時候再考慮向量積，從集合的角度，向量積和向量間的夾角$\theta$的正弦值有關：$\left\|\vec{\mathbf{a}}\times\vec{\mathbf{b}}\right\| = \left\|\vec{\mathbf{a}}\right\|\left\|\vec{\mathbf{b}}\right\|sin\theta$，這就意味著，向量的向量積反應了兩個向量的正交程度（正交：向量積等於零），$sin\theta = 0$ 正交程度最小，$sin\theta = 1$，正交程度最大。

##### 矩陣的向量積

*矩陣乘法*
我們可以將向量理解成一個$1 \times n$的矩陣，當矩陣$A$的列數與向量$\vec x$的分量數目想等式，矩陣和向量的積有定義：

$\underset{m\times n}{A}\vec{\mathbf{x}}=\begin{bmatrix}a_{11} & a_{12} & \ldots & a_{1n} \\ a_{21} & a_{22} & \ldots & a_{2n} \\ \ldots \\ a_{m1} & a_{m2} & \ldots & a_{mn}\end{bmatrix}\begin{bmatrix}x_1 \\ x_2 \\ \ldots \\ x_n \end{bmatrix} = \begin{bmatrix}a_{11}x_1 + a_{12}x_2 + \ldots + a_{1n}x_n \\ a_{21}x_1 + a_{22}x_2 + \ldots + a_{2n}x_n \\ \ldots \\ a_{m1}x_1 + a_{m2}x_2 + \ldots + a_{mn}x_n \\ \end{bmatrix}$

例：$\begin{bmatrix}4 & 3 & 1 \\ 1 & 2 & 5\end{bmatrix}  \begin{bmatrix}5 \\ 2 \\ 7\end{bmatrix} = \begin{bmatrix}4\cdot 5 + 3\cdot 2 + 1\cdot 7 \\ 1 \cdot 5 + 2 \cdot 2 + 5 \cdot 7\end{bmatrix} = \begin{bmatrix}33 \\ 44\end{bmatrix}$

矩陣的向量積，可以當作時矩陣的所有列向量的線性組合：

$\underset { m\times n }{ \mathbf{A} } \vec { \mathbf{x} } =\begin{bmatrix} \underbrace { \begin{bmatrix} a_{ 11 } \\ a_{ 21 } \\ \ldots \\ a_{ m1 } \end{bmatrix} }_{ \vec { \mathbf{ V }_{ 1 } }  }  & \underbrace { \begin{bmatrix} a_{ 12 } \\ a_{ 22 } \\\ldots  \\ a_{ m2 } \end{bmatrix} }_{ \vec { \mathbf{ V_{ 2 } } }  } & \ldots & \underbrace { \begin{bmatrix} a_{ 1n } \\ a_{ 2n } \\ \ldots \\ a_{ mn } \end{bmatrix} }_{ \vec { \mathbf{ V_{ n } } }  }  \end{bmatrix}\begin{bmatrix} x_{ 1 } \\ x_{ 2 } \\ \ldots \\ x_{ n } \end{bmatrix}=x_1\vec{\mathbf{V}_1}+x_2\vec{\mathbf{V}_2}+\ldots+x_n\vec{\mathbf{V}_n}$

而向量$\vec{x}$的每一個份量可以看成$\mathbf{A}$的每一個列向量的加權。**一個矩陣其實就是一個線性變換，一個矩陣乘以一個向量後得到的向量，其實就相當於將這個向量進行了線性變換。**

### 向量的轉置

向量$ \vec{\mathbf{V}} = \underbrace{\begin{bmatrix}v_1 \\ v_2 \\ \ldots \\ v_n \end{bmatrix}}_{n\times 1}$ 的轉置定義為$\mathbf{V}^T = \underbrace{\begin{bmatrix}v_1 & v_2 & \ldots & v_n \end{bmatrix}}_{1 \times n}$

```python
# we use V = (2,4) to give a example
import numpy as np

vec_3 = np.array([[2,4]]) # if we use double bracket, we are writing a two dimentional vecotr
print(vec_3.T) # print the transport of vec_3
```

向量的轉置有一個性質：一個向量$\vec {v}$ 點乘另一個向量$\vec{w}$，其結果和向量$\vec{v}$轉置後和向量$\vec{w}$做矩陣乘法相同。即：$\vec{\mathbf{v}} \cdot \vec{\mathbf{w}} = \vec{\mathbf{v}}^T \vec{\mathbf{w}}$

## Chapter Four Vector Space

### 張成空間（span）

一組向量的張成空間，說白了就是指這些向量隨便線性組合後能夠表示多少個向量，也被稱為線性生成空間。記做$span(S)$；如果$S = \{v_1, v_2, \dots, v_n\}$是 $V$的有限子集，則生成空間為：$span(S) = span(v_1, v_2, \dots, v_n) = \{\lambda_1 v_1, \lambda_2 v_2, \dots, \lambda_n v_n|\lambda_1, \lambda_2, \dots, \lambda_n \in K\}$

例如，對於$\mathbf{R}^2$空間中兩個不平行的非零向量$\vec{vec_4} = \begin{bmatrix} 2 \\ 1 \end{bmatrix}$和向量$\vec{vec_5} = \begin{bmatrix} 0 \\ 3 \end{bmatrix}$，不難發現，這兩個向量可以表示二維空間$\mathbf{R}^2$中的任意一個向量，即，$span(\vec{vec_4}, \vec{vec_5}) = \mathbf{R}^2$，證明如下：

對於$\mathbf{R}^2$中任意向量$\begin{bmatrix} x \\ y \end{bmatrix}$，假設可以用$vec_4$和$vec_5$線性組合而成，

那麼有：$c_1 \begin{bmatrix}2 \\ 1\end{bmatrix} + c_2 \begin{bmatrix} 0 \\ 3 \end{bmatrix} = \begin{bmatrix} x \\ y \end{bmatrix}$

即：$\left\{
\begin{align}
c_1 \cdot 2 & + c_2 \cdot 0 &= x\\\
c_1 \cdot 1 & + c_2 \cdot 3 &= y
\end{align}
\right.$

求解得：
$\left\{
\begin{align}
c_1 &= \frac{x}{2}\\
c_2 &= \frac{y}{3} - \frac{x}{6}
\end{align}
\right.$

由於$x, y$的值已經確定，所以$c_1，c_2$的值也必然是唯一的。

### 線性相關和線性無關（Linearly Dependent and Linearly Independent）

在線性代數裡，向量空間的一組元素中，若沒有向量可用有限個其他向量的線性組合所表示，则稱為線性無關或線性獨立（linearly independent），反之稱為線性相關（linearly dependent）。例如在三維歐幾里得空間$\mathbf{R}^3$的三個向量$(1, 0, 0)，(0, 1, 0)$ 和$(0, 0, 1)$線性無關。但$(2, −1, 1)，(1, 0, 1)$和 $(3, −1, 2)$線性相關，因為第三個是前兩個的和。

*相信各位都已知道那八條還是十條相關性，這裡不做贅述*

上面那個例子例如，$\vec{vec_4} = \begin{bmatrix} 2 \\ 1 \end{bmatrix}$和向量$\vec{vec_5} = \begin{bmatrix} 0 \\ 3 \end{bmatrix}$，如果給他們再加一個向量$\vec{vec_6} = \begin{bmatrix} 5 \\ 2 \end{bmatrix}$，由於$\vec{vec_5}$可以由其他兩個向量線性組合而成，由三個向量共同張成的空間並沒有發生變化，仍然是$\mathbf{R}^3$，因此稱集合$\{\vec{vec_3}, \vec{vec_4}, \vec{vec_5}\}$線性相關。

### 如何判斷是否線性相關

一個向量集合$s = v_1, v_2, \dots, v_n$ 線性相關的充分必要是存在一部分非零係數使得：$c_1v_1 + c_2v_2 + \dots + c_nv_n = \begin{bmatrix} 0 \\ 0 \\ \dots \\ 0 \end{bmatrix}$。

例如：向量：$\begin{bmatrix}2 \\ 1\end{bmatrix}$ 和 $\begin{bmatrix}3 \\ 2\end{bmatrix}$，則先寫出如下等式：
$c_1 \begin{bmatrix}2 \\ 1\end{bmatrix} + c_2 \begin{bmatrix}3 \\ 2\end{bmatrix} = \begin{bmatrix}0 \\ 0\end{bmatrix}$,

容易求解得：$\begin{bmatrix}c_1 \\ c_2\end{bmatrix} = \begin{bmatrix}0 \\ 0\end{bmatrix}$，說明這兩個向量線性無關。

類似的，對於三個$\mathbf{R}^3$中的向量$\begin{bmatrix}2 \\ 0 \\ 0\end{bmatrix}, \begin{bmatrix} 0 \\ 1 \\ 0\end{bmatrix}$和$\begin{bmatrix} 0 \\ 0 \\ 7\end{bmatrix}$，不難判斷出它們是線性無關的，它們共同張成了$\mathbf{R}^3$空間。

而對於向量集合：$\left\{\begin{bmatrix}2 \\ 1\end{bmatrix}, \begin{bmatrix}3 \\ 2\end{bmatrix}, \begin{bmatrix}1 \\ 2 \end{bmatrix}\right\}$，不難算出存在非零係數：$\begin{bmatrix}c_1 \\ c_2 \\ c_3\end{bmatrix} = \begin{bmatrix}-4 \\ 3 \\ -1\end{bmatrix}$ 使得 $c1 \begin{bmatrix}2 \\ 1\end{bmatrix} + c_2 \begin{bmatrix}3 \\ 2\end{bmatrix} + c_3 \begin{bmatrix}1 \\ 2 \end{bmatrix} = \begin{bmatrix}0 \\ 0\end{bmatrix}$。因此，此集合線性相關。

## Chapter Five Linear Algebra Advanced Text

在前面幾張，已經簡單介紹了線性代數中最基本的數據表達方式：**矩陣**和**向量**。有了這兩個數學工具作為基礎，我們可以進一步討論下面的內容：

1. 如何理解線性代數的基。

2. 向量中的子空間、另空間、列空間、行空間、零空間都是什麼？我們應該如何求解？

3. 如何用線性代數的知識來擬合數據？

4. 機器學習、圖形處理中常見的“特徵向量”究竟是什麼？它和變換矩陣有什麼關係？

### 階梯形矩陣（Echelon Matrix）

階梯形矩陣是一類非常實用的工具，可以幫助我們求解線性空間的基，這就是能用在諸如計算解不是唯一的方程組之類的問題上。

#### 階梯形矩陣

若矩陣$\mathbf{A}$滿足如下條件：

1. 若有零行（元素全為零的行），則零行應在最下方；

2. 非零首元（即非零行的第一個不為零的元素）的列標號隨行標號的增加而嚴格遞增。

則這個矩陣稱為階梯形矩陣。

示例：

$\begin{bmatrix}
2 & 0 & 2 & 1 \\
0 & 5 & 2 & -2 \\
0 & 0 & 3 & 2 \\
0 & 0 & 0 & 0
\end{bmatrix}$

#### 行簡化階梯形矩陣（Row Echelon Matrix）

若矩陣$\mathbf{A}$滿足如下條件：

1. 它是階梯形矩陣;

2. 非零首元所在列出了非零首元外，其餘元素都為零。

則稱此矩陣為行簡化階梯形矩陣。

示例：

$\begin{bmatrix}
2 & 0 & 2 & 1 \\
0 & 5 & 2 & -2 \\
0 & 0 & 3 & 2 \\
0 & 0 & 0 & 0
\end{bmatrix}$

#### 行最簡階梯形矩陣（reduced row echelon form）

若矩陣$\mathbf{A}$滿足如下條件：

1. 它是行簡化階梯形矩陣；

2. 非零首元都為一。

則稱它為行最簡階梯形矩陣。下面我們來講講如何將矩陣化簡為最簡階梯形：

對於這個矩陣$\mathbf{A} = \begin{bmatrix}
1 & 2 & 1 & 1 & 7\\
1 & 2 & 2 & -1 & 12\\
2 & 4 & 0 & 6 & 4
\end{bmatrix}$，使用**初等變換（Elementary Operations of Linear System）**就可以將矩陣轉換成如下形式：
$\begin{bmatrix}
1 & 2 & 1 & 1 & 7\\
1 & 2 & 2 & -1 & 12\\
2 & 4 & 0 & 6 & 4
\end{bmatrix}
\rightarrow
\begin{bmatrix}
1 & 2 & 1 & 1 & 7\\
0 & 0 & 1 & -2 & 5\\
2 & 4 & 0 & 6 & 4
\end{bmatrix}
\rightarrow
\begin{bmatrix}
1 & 2 & 1 & 1 & 7\\
0 & 0 & 1 & -2 & 5\\
0 & 0 & -2 & 4 & -10
\end{bmatrix}
\rightarrow
\begin{bmatrix}
1 & 2 & 1 & 1 & 7\\
0 & 0 & 1 & -2 & 5\\
0 & 0 & 0 & 0 & 0
\end{bmatrix}
\rightarrow
\begin{bmatrix}
1 & 2 & 0 & 3 & 2\\
0 & 0 & 1 & -2 & 5\\
0 & 0 & 0 & 0 & 0
\end{bmatrix}$

**行最簡形**非常實用，可以用來解方程（滑稽；例如下面這個方程組：

$\left\{
\begin{eqnarray}
x_1 + 2x_2 + x_3 + x_4 &=& 7 \\\
x_1 + 2x_2 + 2x_3 - x_4 &=& 12 \\\
2x_1 + 4x_2 + 6x_4 &=& 4
\end{eqnarray}
\right.$

按照常理，只有三個方程，肯定無法求解出四個未知數，此時，如果我們使用`numpy.linalg.solve`，求解時就會觸發`LinAlgError`，但是通過化成最簡形，我們可以進一步找出變量的憲制關係。首先，將方程組表達成增廣矩陣形式：

$\begin{bmatrix}
1 & 2 & 1 & 1 & 7\\
1 & 2 & 2 & -1 & 12\\
2 & 4 & 0 & 6 & 4
\end{bmatrix}$

然後將其行簡化，得：

$\begin{bmatrix}
1 & 2 & 0 & 3 & 2\\
0 & 0 & 1 & -2 & 5\\
0 & 0 & 0 & 0 & 0
\end{bmatrix}$

從結果可以看出這個矩陣的主列有兩個，而且是線性無關的。所以矩陣$A$的秩是$2$，，即，$rank\mathbf{(A)} = 2$.

**矩陣的秩的一個特性：矩陣$A$等於矩陣$A$的轉置地秩序，即，**$rank(\mathbf{A})=rank(\mathbf{A^T})$

在`python`中，可以使用`numpy`包中的`linalg.matrix_rank`方法計算矩陣的秩：

```python
import numpy as np
a = np.matrix('1 1 1 1;1 2 3 4;4 3 2 1')
print(np.linalg.matrix_rank(a))
```

**注意**⚠，在`numpy`中的秩和線性代數裡的秩不是同一個概念。在`numpy`中維度（dimensions）叫做軸（axes），軸的個數叫做秩。

```python
import numpy as np
a = np.matrix('1 1 1 1;1 2 3 4; 0 0 1 0')
print (a.ndim)  # 2（维度）
print (np.rank(a))  # 2（a.ndim 的别名，已经过时）
print (np.linalg.matrix_rank(a))  # 3（秩）
```

#### 行空間

有了列空間的定義，行空間顧名思義其實就是矩陣每一行轉置得到的向量張成的自空間，也就是矩陣的轉置的列空間，記做：$R(\mathbf{A}) = C(\mathbf{A}^T)$

例如：矩陣$\mathbf{A} = \begin{bmatrix}1 & 1 & 1 & 1 \\ 1 & 2 & 3 & 4 \\4 & 3 & 2 & 1\end{bmatrix}$的行空間就等於：$R(\mathbf{A}) = C(\mathbf{A}^T) = span\left(\begin{bmatrix}1 \\ 1 \\ 1 \\ 1\end{bmatrix}\begin{bmatrix}1 \\ 2 \\ 3 \\ 4\end{bmatrix}\begin{bmatrix}4 \\ 3 \\ 2 \\ 1\end{bmatrix}\right)$

#### 零空間

上面已經講過了零空間，現在這裡的零空間是指矩陣$A^T$的零空間，也就是**左零空間**。即：$N(\mathbf{A}^T) = \left\{ \vec{\mathbf{x}} | \mathbf{A}^{T} \vec{\mathbf{x}} = \vec{\mathbf{0}} \right\} = \left\{ \vec{\mathbf{x}} | \vec{\mathbf{x}}^{T} \mathbf{A} = \vec{\mathbf{0}}^{T} \right\}$

例如，矩陣$\mathbf{B} = \begin{bmatrix}1 & 1 & 4 \\ 1 & 2 & 3 \\1 & 4 & 2\\ 1 & 3 & 1\end{bmatrix}$的轉置是矩陣$\mathbf{A} = \mathbf{A} = \begin{bmatrix}1 & 1 & 1 & 1 \\ 1 & 2 & 3 & 4 \\4 & 3 & 2 & 1\end{bmatrix}$，因此左零空間等於：$N(\mathbf{B^T}) = N(\mathbf{A}) = span\left(\begin{bmatrix} 1 \\ -2 \\ 1 \\ 0 \end{bmatrix} \begin{bmatrix} 2 \\ -3 \\ 0 \\ 1 \end{bmatrix}\right)$；由於轉置是對稱的，所以矩陣$A$的轉置的左零空間也是矩陣$A$的零空間。

#### 子空間的正交補

假設$V$是$R^n$的一個子空間，那麼$V$的正交補$V^\perp$也是一個子空間，定義為$\left\{\vec{\mathbf{x}} | \vec{\mathbf{x}} \vec{\mathbf{v}}=0\right\}$，也就是說$R^n$中所有正交於$V$的向量所組成的子空間。

由於正交是對稱的，所以正交補也是對稱的。一個子空間的正交補的正交補依然等於這個子空間。

矩陣的零空間是行空間的正交補：$N(\mathbf{A}) = R(\mathbf{A})^{\bot}$；反過來，矩陣的左零空間是列空間的正交補：$N(\mathbf{B}^T) = C(\mathbf{B})^{\bot}$

### 最小二乘逼近

最小二乘法是一個使用的數學工具，利用它在方程誤解的情況下給出近似解。在機器學習中，最小二乘逼近是一個重要的擬合方法。

假設有一個方程：$\underset{n\times k}{\mathbf{A}}\vec{\mathbf{x}} = \vec{\mathbf{b}}$

無解。把上式寫成：$\vec{a_1}\vec{\mathbf{x}} + \vec{a_2}\vec{\mathbf{x}} + \ldots + \vec{a_k}\vec{\mathbf{x}} = \vec{\mathbf{b}}$

無解，就意味著$\mathbf{A}$的所有列向量的張成空間不包括向量$\vec{b}$。即：$\vec{\mathbf{b}} \notin span(C(\mathbf{A}))$

我們可以通過最小二乘法，求解出近似解。即，要找出一些$\vec{x*}$使得$\left\|\vec{\mathbf{b}}-\mathbf{A}\vec{\mathbf{x}^*}\right\|$最小。用向量$\vec{\mathbf{V}}$代表$\mathbf{A}\vec{\mathbf{x*}}$，有：

$\left\|
\begin{bmatrix}
\vec{b_1}-\vec{v_1}\\
\vec{b_2}-\vec{v_2}\\
\ldots\\
\vec{b_n}-\vec{v_n}\\
\end{bmatrix}
\right\|^2
=
(b_1-v_1)^2 + (b_2-v_2)^2 + \ldots + (b_n-v_n)^2$

把這個值最小化的過程叫做**最小二乘逼近**。

如何求出$\mathbf{A}\vec{\mathbf{x*}}$這個近似值呢？從集合上考慮，列空間可以看成空間中張成的一個平面，而平面$\vec{b}$並不落在這個平面上。但我們知道，在這個平面上與向量$\vec{b}$最接近的向量就是它的投影！所以，$\mathbf{A}\vec{\mathbf{x}^*} = Proj_{C(\mathbf{A})}\vec{\mathbf{b}}$，直接計算$Proj_{C(\mathbf{A})}\vec{\mathbf{b}}$並不簡單。不過，$\vec{\mathbf{b}}-\mathbf{A}\vec{\mathbf{x}}$其實就是$\mathbf{A \vec{x}}$的正交補，所以一個簡單的求解方法是將原來無解的方程做成一個$\mathbf{A}$的轉置再求解：$\mathbf{A}^T\mathbf{A}\vec{\mathbf{x}^*} = \mathbf{A}^T\vec{\mathbf{b}}$得出的解就是原方程的近似解。

#### 示例 解下列方程組
$\left\{
\begin{eqnarray}
x + y &=& 3 \\\
x - y &=& -2 \\\
y &=& 1
\end{eqnarray}
\right.$

將三個方程表示的直線畫出來，可以看出這三條直線並沒有交點：

```python
import numpy as np
import matplotlib.pyplot as plt
from pylab import *

plt.xlim(-3,3)
plt.ylim(0,6)
plt.plot([-2,3],[0,5],'blue',[-3,3],[6,0],'yellow',[-3,3],[1,1],'red')
```

那麼，我們應該如何找出一個與三條直線距離最近的一個點呢？這時候就可以使用最小二乘逼近了。

先將方程寫成矩陣和向量的形式：

$\underbrace{
\begin{bmatrix}
1 & 1 \\
1 & -1 \\
0 & 1
\end{bmatrix}
}_{\mathbf{A}}
\underbrace{
\begin{bmatrix}
x \\
y
\end{bmatrix}
}_{\vec{\mathbf{x}}}
=
\underbrace{
\begin{bmatrix}
3 \\
-2 \\
1
\end{bmatrix}
}_{\vec{\mathbf{b}}}$

這個等式的最小二乘逼近就是：

$\begin{align}
\begin{bmatrix}
1 & 1 & 0 \\
1 & -1 & 1\\
\end{bmatrix}
\begin{bmatrix}
1 & 1 \\
1 & -1 \\
0 & 1
\end{bmatrix}
\begin{bmatrix}
x^* \\
y^*
\end{bmatrix}
& =
\begin{bmatrix}
1 & 1 & 0 \\
1 & -1 & 1\\
\end{bmatrix}
\begin{bmatrix}
3 \\
-2 \\
1
\end{bmatrix}
\\\
\begin{bmatrix}
2 & 0 \\
0 & 3
\end{bmatrix}
\begin{bmatrix}
x^* \\
y^*
\end{bmatrix}
& =
\begin{bmatrix}
1 \\
6
\end{bmatrix}
\end{align}$

由於是二解方程，我們可以很容易求出矩陣$\begin{bmatrix}2 & 0 \\ 0 & 3\end{bmatrix}$的逆是：$\begin{bmatrix}\frac{1}{2} & 0 \\ 0 & \frac{1}{3}\end{bmatrix}$

所以：

$\begin{bmatrix}
x^* \\
y^*
\end{bmatrix}
=
\begin{bmatrix}\frac{1}{2} & 0 \\ 0 & \frac{1}{3}\end{bmatrix}
\begin{bmatrix}
1 \\
6
\end{bmatrix}
=
\begin{bmatrix}
\frac{1}{2} \\
2
\end{bmatrix}$

在`python`中，可以使用`numpy.linalg.lstsq`方法來求解最小二乘逼近。

```python
import numpy as np
import matplotlib.pyplot as plt
from pylab import *

x = [0.5]
y = [2]
plt.xlim(-3,3)
plt.ylim(0,6)
plt.plot([-2,3],[0,5],'blue',[-3,3],[6,0],'yellow',[-3,3],[1,1],'red',xData,yData1,'black')
plt.scatter(x,y,color = 'black')
```

```python
a = np.array([[1, 1], [1, -1], [0, 1]])
b = np.array([3, -2, 1])
x = np.linalg.lstsq(a,b)
print(x)
```

`numpy.inalg.lstsq`的返回值包括了四個部分：

1. 最小二乘逼近，如果`b`是二維的，那麼這個逼近的結果有多個列，每一列是一個逼近解，逼近解就是$\begin{bmatrix}0,5 \\ 2 \end{bmatrix}$.

2. 殘差。即，每一個`b - a*x`的長度的和。對於上例，殘差是$1.5$.

3. 矩陣$a$的秩。對於上例，矩陣$a$的秩為$2$。

4. 矩陣$a$的歧異值。對於上例，矩陣$a$的奇異值為：$\begin{bmatrix}1.73205081 \\  1.41421356\end{bmatrix}$

#### 线性回归

問題：給定$4$個坐標點$(-1,0),(0,1),(1,),(2,1)$，求一條經過這些點的直線$y = mx + b$。

將四個點畫圖如下：

```python
import numpy as np
import matplotlib.pyplot as plt
from pylab import *

x = [-1,0,1,2]
y =[0,1,2,1]

plt.scatter(x,y,color = 'black')
```

顯然這樣的直線並不存在。然而我們能夠使用最小二乘逼近，找到一條盡可能靠近這些點的直線。將四個點表示成方程組的形式：

$\left\{
\begin{eqnarray}
f(-1) &= -m + b = 0\\\
f(0) &= 0 + b  = 1\\\
f(1) &= m + b = 2\\\
f(2) &= 2m + b = 1
\end{eqnarray}
\right.$

然後方程組用矩陣和向量的形式：

    $\underbrace{
\begin{bmatrix}
-1 & 1 \\
0 & 1 \\
1 & 1 \\
2 & 1
\end{bmatrix}
}_{\mathbf{A}}
\underbrace{
\begin{bmatrix}
m\\
b
\end{bmatrix}
}_{\vec{\mathbf{x}}}
=
\underbrace{
\begin{bmatrix}
0\\
1\\
2\\
1
\end{bmatrix}
}_{\vec{\mathbf{b}}}$

這兩個等式的最小二乘逼近就是：

    $\begin{align}
\begin{bmatrix}
-1 & 0 & 1 & 2 \\
1 & 1 & 1 & 1
\end{bmatrix}
\begin{bmatrix}
-1 & 1 \\
0 & 1 \\
1 & 1 \\
2 & 1
\end{bmatrix}
\begin{bmatrix}
m^*\\
b^*
\end{bmatrix}
&=
\begin{bmatrix}
-1 & 0 & 1 & 2 \\
1 & 1 & 1 & 1
\end{bmatrix}
\begin{bmatrix}
0\\
1\\
2\\
1
\end{bmatrix}\\\
\begin{bmatrix}
6 & 2 \\
2 & 4
\end{bmatrix}
\begin{bmatrix}
m^*\\
b^*
\end{bmatrix}
&=
\begin{bmatrix}
4\\
4
\end{bmatrix}
\end{align}$

容易求得$\begin{bmatrix}6 & 2\\2 & 4\end{bmatrix}$的逆$\frac{1}{20}\begin{bmatrix}4 & -2\\-2 & 6\end{bmatrix}$，因此

    $\begin{bmatrix}m^*\\b^*\end{bmatrix} = \frac{1}{20}\begin{bmatrix}4 & -2\\-2 & 6\end{bmatrix}\begin{bmatrix}4 \\ 4\end{bmatrix} = \frac{1}{20}\begin{bmatrix}8 \\ 16\end{bmatrix} = \begin{bmatrix}\frac{2}{5} \\ \frac{4}{5}\end{bmatrix}$

將直線$y = \frac{2}{5}x + \frac{4}{5}$繪圖如下所示：

```python
import numpy as np
import matplotlib.pyplot as plt
from pylab import *

x = [-1,0,1,2]
y =[0,1,2,1]
x1 = np.linspace(-3,3)
y1 = lambda x: 2*x/5 + 4/5
plt.scatter(x,y,color = 'black')
plt.plot(x1,y1(x1),'r')
```

這就是所求的直線的近似解。

```python
a = np.matrix('-1 1;0 1;1 1;2 1')
b = np.array([0, 1, 2, 1])
x = np.linalg.lstsq(a, b)
print(x)
```

### 特徵向量（Elgen Vector）

“特徵”在模式識別和圖像處理中是一個非常常見的詞彙，我們要認識和描繪一件事物，首先就要找出這個事務的特徵。同樣的道理，要讓計算機識別一件事物，稍嫌就要讓計算機學會理解或者抽象出食物的特徵。什麼樣的東西能當成特徵呢？那必須是能“放之四海而皆準”的依據。不論個體如何變幻，都能從中吵到這類群裡共有的特點。

在線性代數中，“特徵”就是一個更抽象的描述。我們知道，矩陣乘法對應了一個變換，是把任意一個向量變換成裡一個方向或長度都大多不同的新向量。在這個變幻的過程中，原向量主要發成旋轉、伸縮的變化。如果矩陣對某一個向量或某些向量只發生伸縮（尺度）變換，而沒有產生旋轉的效果（也就意味著張成的子空間發生改變），這樣的向量就被認為是特徵向量。

    $\mathbf{T}(\vec{\mathbf{v}}) = \underbrace{\mathbf{A}}_{n\times n}\vec{\mathbf{v}} = \underbrace{\lambda}_{特征值} \overbrace{\vec{\mathbf{v}}}^{特征向量}$

其中，$\mathbf{T}$是一種線性變換，我們知道線性變換，我們知道線性變換可以用矩陣向量積來表示，因此可以表示成$\mathbf{A \vec{v}}$。$\mathbf{A}$是一個$n \times n$的方陣。$\vec{v}$就是特徵向量（Elgen Vector），也就是「能被伸縮的向量」（要求是非$0$向量），而$\lambda$的特徵向量$\vec{v}$所對應的特徵之，也就是「伸縮了多少」。如果特徵之是負數，那說明了矩陣不但吧向量拉長（縮短）了，而且讓向量只想了相反的方向。

**簡而言之，特徵就是在線性變量當中不變的向量**

#### 求解特徵值

非$0$向量$\vec{v}$是線性變化矩陣$\mathbf{A}$的特徵向量，需要滿足如下條件：

    $det(\lambda \mathbf{I}_n - \underbrace{\mathbf{A}}_{n\times n}) = 0$

其中，$det$表示矩陣行列式，$\lambda$是特徵值，$\mathbf{I}$是單位矩陣。

例如矩陣$\mathbf{A} = \begin{bmatrix}1 & 2 \\ 4 & 3\end{bmatrix}$，代入公式$2$得：

    $\begin{align} det\left( \lambda \begin{bmatrix} 1 & 0 \\ 0 & 1 \end{bmatrix}-\begin{bmatrix} 1 & 2 \\ 4 & 3 \end{bmatrix} \right)  &=0 \\ det\left( \begin{bmatrix} \lambda  & 0 \\ 0 & \lambda  \end{bmatrix}-\begin{bmatrix} 1 & 2 \\ 4 & 3 \end{bmatrix} \right)  &=0 \\ det\left( \begin{bmatrix} \lambda -1 & -2 \\ -4 & \lambda -3 \end{bmatrix} \right)  &=0 \end{align}$

所以有：

    $\begin{align} (\lambda -1)(\lambda -3)-8 & =0 \\ \lambda ^{ 2 }-4\lambda -5 &=0 \\ (\lambda - 5)(\lambda +1) &= 0\end{align}$

因此$\lambda$得值等於$5$或者$-1$。

在`python`中，可以使用`numpy.linalg.eigvals`方法求解一個方陣的特徵值：

```python
a = np.matrix('1 2;4 3')
print(np.linalg.eigvals(a))
```

```python
b = np.matrix('1 2 3;4 3 1')#如果矩陣不是方針，就會出現 LinalgError 錯誤
print(np.linalg.eigvals(b) )
```

#### 求解特徵向量

變換矩陣$\mathbf{A}$的特徵空間（特徵向量的空間）可以用下面的等式來求解：

    $\mathbf{E}_{\lambda}=N(\lambda I_n - \mathbf{A})$

例如上面的變換矩陣$\mathbf{A} = \begin{bmatrix}1 & 2 \\ 4 & 3 \end{bmatrix}$，帶入公式$3$得：

    ${ E }_{ \lambda  }=N\left( \lambda I_{ n }-\begin{bmatrix} 1 & 2 \\ 4 & 3 \end{bmatrix} \right) =N\left( \lambda \begin{bmatrix} 1 & 0 \\ 0 & 1 \end{bmatrix}-\begin{bmatrix} 1 & 2 \\ 4 & 3 \end{bmatrix} \right) =N\left( \begin{bmatrix} \lambda -1 & -2 \\ -4 & \lambda -3 \end{bmatrix} \right)$

當$\lambda = 5$時，

    ${ E }_{ 5  }=N\left( \begin{bmatrix} 4 & -2 \\ -4 & 2 \end{bmatrix} \right)$

利用前面所學的零空間的求解方法，得：

    ${ E }_{ 5  }= span\left(\begin{bmatrix}\frac{1}{2} \\ 1 \end{bmatrix}\right)$

同樣的，當$\lambda = -1$時，

    ${ E }_{ -1  }= span\left(\begin{bmatrix}1 \\ -1 \end{bmatrix}\right)$

在`python`中，可以使用`numpy.linalg.eig`方法來求解方針的特徵值和特徵向量：

```python
a = np.matrix('1 2;4 3')
print(np.linalg.eig(a))
```

得到的元祖中，第一部分時特徵值，和前面使用`numpy.linalg.eigvals`得到的結果完全一樣；第二部分時特徵向量，乍一看和我們上面求解的結果不一樣，但如果我們這麼些就完全不一樣了：

    $\begin{bmatrix}-0.70710678\begin{bmatrix}1 \\ -1\end{bmatrix} & -0.89442719\begin{bmatrix}\frac{1}{2} \\ 1\end{bmatrix} \end{bmatrix}$

變換矩陣線性無關的特徵向量特別適合作為空間的基，因為在這些方向上變換矩陣可以拉伸向量而不必扭曲和旋轉它，使得計算大為簡單。我們把這種基稱為**特徵基**

jupyterfiles/Use PY in Linear Algebra.ipynb


#
 Transmission function

Usually, XPS measurements are performed with the constant energy of detected electrons inside of the analyzer (fixed analyzer transmission mode). Electrons of a specific initial kinetic energy $E_k$ are measured by setting voltages for the lens system that both focus onto the entrance slit the electrons of the required initial energy and retards their velocity so that their kinetic energy after passing through the transfer lenses matches the pass energy $E_p$ of the analyser.

The measured intensity depends on the collection efficiency of the receiver, i.e., on its transmission function $T(E_k, E_p)$ (=intensity/energy response function). $T(E_k, E_p)$ includes all instrumental parameters of the XPS receiver such as acceptance area, acceptance solid angle and the detection efficiency. Thus $T(E_k, E_p)$ is instrument dependent [Trigueiro et al 2018].

There are several methods for measuring $T(E_k, E_p)$, which are decribed in details by Weng et al 1993. In case of liquid jet XPS the most suitable method is one developed by Carrazza and León 1991. According to this method, $T(E_k, E_p)$ dependence can be evaluated from:

\begin{equation}
{T \sim E_p (\frac{E_p}{E_k})^{n}}
\end{equation}

where $n$ - coefficient depending on $E_k$, which needs to be found. To evaluate it from the experiment, at least two measurements of the same sample at the same $E_k$ should be performed. The expression for total recorded photoelectron signal $I$ from a specie is:

\begin{equation}
  I \sim \alpha F \sigma T(E_k, E_p) I_n
\end{equation}

where $\alpha$ is an experimental alignment factor, $F$ is total photon flux, $\sigma$ is photoionization cross-section, $I_n$ is normalized signal. Assuming that:

$\alpha$ is constant during the experiment

total photon flux can be substituted by mirror current $I_m$ due to their proportionality

$I_n$ is equal for 2 measurements of the same sample at the same $E_k$

$I_n$ and $I$ can be substituted by peak areas $A_n$ and $A$

we can deduce from the ratio of $I_n$ for 2 measurements and from the expression for $T(E_k, E_p)$ the coefficient $n$:

\begin{equation}
  n = \log{\frac{A_1 I_{m2} E_{p2}}{A_2 I_{m1} E_{p1}}} / \log{\frac{E_{p1}}{E_{p2}}}
\end{equation}

The advantage of the method is that a knowledge of $\alpha$ and $\sigma$ nor the use of a contamination-free sample is not required.

Let's take an example in Python code. 3 pairs of measurements were performed on a sample with varying $E_k$ of 60 (5 and 20 eV $E_p$), 210 and 410 eV (20 and 50 eV $E_p$ in each case):

```python
# Data obtained. These can be substituted by your data.
A_1 =  [213132,  1409318, 373099]  #
A_2 =  [741699,  3995173, 1068549] #
I_m1 = [243.861, 410.884, 438.056] # mA
I_m2 = [245.856, 430.716, 437.926] # mA
E_p1 = [5,       20,      20]      # eV
E_p2 = [20,      50,      50]      # eV

import numpy as np # import library

n = [] # array for transmisson coefficient
# fill n array
for i in range(len(A_1)):
    coef_1 = (A_1[i] * I_m2[i] * E_p2[i]) / (A_2[i] * I_m1[i] * E_p1[i])
    coef_2 = E_p1[i] / E_p2[i]
    n.append(abs(
        np.log(coef_1) / np.log(coef_2)
                )
            )
# average n
n_aver = round(sum(n) / len(n), 2)

print('n values: ')
[print(round(i, 2), end =' ') for i in n]
print(f'\nAverage: {n_aver}')
```

    n values:
    0.11 0.09 0.15
    Average: 0.11

```python
# now let's make plots
# more libraries
import matplotlib.pyplot as plt
from matplotlib import rc

# choosing font and text style for plots
rc('font',**{'family':'serif',
             'size':16})
rc('text', usetex=True) # use LaTeX
fig, ax = plt.subplots() # create plot space

ek = np.arange(5, 500, 1) # kinetic energy
ep = sorted(list(set(E_p1 + E_p2))) # pass energy used
for energy in ep:
    ax.plot(ek, (lambda ek: energy * (energy / ek) ** n_aver)(ek))

ax.set_ylabel('Transmisson [a.u.]')
ax.set_xlabel('Ek [eV]')
ax.grid()
# ax.grid(title=r"")
plt.show();
```


```python
import numpy as np
import pandas as pd
from pylab import *
from sympy import Symbol,expand
%config InlineBackend.figure_formats = ['svg']

x=Symbol('x')
```

```python
import pathlib
if not pathlib.Path("Generator_Data_ElisabetaTedeschi.xlsx").exists():
  !curl -O https://raw.githubusercontent.com/joaochenriques/MCTE_2020_2021/main/Barrages_Turbine_Generator_Maps/Generator_Data_ElisabetaTedeschi.xlsx
```

      % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                     Dload  Upload   Total   Spent    Left  Speed
    100  8684  100  8684    0     0  39294      0 --:--:-- --:--:-- --:--:-- 39294

```python
def LSQ_polyfit( NP, XL, XC_fun, XC_der ):

    ########################################################
    # LINEAR LEAST SQUARES PROBLEM
    #
    ( xl, yl ) = XL

    # Vandermonde matrix
    X = xl[:, None] ** np.arange( NP, -1, -1 )

    XTX = X.transpose().dot( X )
    XTy = X.transpose().dot( yl )

    # NO CONSTRAINTS IMPOSED => standard LSQ solution
    if XC_fun == None and XC_der == None:
      AA = XTX
      bb = XTy
      return np.linalg.solve( AA, bb )

    ########################################################
    # LAGRANGE MULTIPLIERS FOR CONSTRAINTS
    #
    if XC_fun != None:
      ( xc_fun, yc_fun ) = XC_fun

      # function constraints
      xF = np.array( xc_fun )
      bF = np.array( yc_fun )

      # Vandermonde matrix
      AF = xF[:, None] ** np.arange( NP, -1, -1 )

    if XC_der != None:
      ( xc_der, yc_der ) = XC_der

      # derivative constraints
      xD = np.array( xc_der )
      bD = np.array( yc_der )

      # Derivative of the Vandermonde matrix
      AD = np.array( [  (i*xD**(i-1) if i > 0 else 0*xD) \
                        for i in range(NP,-1,-1) ] ).T

    if XC_fun != None and XC_der != None:
      Ac = np.block( [ [AF], [AD] ] )
      bc = np.block( [ bF, bD ] )
    elif XC_fun != None:
      Ac = AF
      bc = bF
    elif XC_der != None:
      Ac = AD
      bc = bD

    AcT = Ac.transpose()
    Z0 = np.zeros( ( Ac.shape[0], Ac.shape[0] ) )

    ########################################################
    # ASSEMBLE AND SOLVE GLOBAL SYSTEM
    #
    AA = np.block( [ [ XTX, AcT ], [ Ac, Z0 ] ] )
    bb = np.block( [ XTy, bc ] )

    alambda = np.linalg.solve( AA, bb )

    # final polynomial coeffs without lambdas
    return alambda[ 0:NP+1 ]
```

```python
dfc = pd.read_excel( 'Generator_Data_ElisabetaTedeschi.xlsx', 'Sheet1' )
Load = np.array( dfc['Load'] )
Eff = np.array( dfc['Eff'] )

Load = np.array( Load[~np.isnan(Load)] )
Eff = np.array( Eff[~np.isnan(Eff)] )
```

```python
Load_1 = Load[:6]
Load_2 = Load[5:]
Eff_1  = Eff[:6]
Eff_2  = Eff[5:]
```

```python
plt.plot( Load_1, Eff_1, 'o' )
plt.plot( Load_2, Eff_2, 'o' )

poly_C1 = LSQ_polyfit( 4, ( Load_1, Eff_1 ), ([Load_1[-1]],[Eff_1[-1]]), None )
Load_1_plt = np.linspace( Load_1[0], Load_1[-1], 100 )
Eff_1_plt = np.polyval( poly_C1, Load_1_plt )
plot( Load_1_plt, Eff_1_plt, '-' )

# impose derivative at curved common point
poly_C1_der = np.poly1d( poly_C1 ).deriv()
der1 = poly_C1_der(Load_2[0])

poly_C2 = LSQ_polyfit( 4, ( Load_2, Eff_2 ), ([Load_2[0]],[Eff_2[0]]), ([Load_2[0]],[der1]) )
Load_2_plt = np.linspace( Load_2[0], 1.0, 100 )
Eff_2_plt = np.polyval( poly_C2, Load_2_plt )
plot( Load_2_plt, Eff_2_plt, '-' )

display( expand(np.poly1d( np.round( poly_C1, 5 ) )(x)) )
display( expand(np.poly1d( np.round( poly_C2, 5 ) )(x)) )
Load_2[0], poly_C1, poly_C2
```

$\displaystyle - 6714.48631 x^{4} + 2591.59775 x^{3} - 380.83406 x^{2} + 27.04232 x + 0.00329$

$\displaystyle - 1.16857 x^{4} + 3.31173 x^{3} - 3.44296 x^{2} + 1.5416 x + 0.71041$

    (0.125423729,
     array([-6.71448631e+03,  2.59159775e+03, -3.80834059e+02,  2.70423225e+01,
             3.29394948e-03]),
     array([-1.16856952,  3.31172525, -3.44296217,  1.5416029 ,  0.71040716]))

```python
subplots_adjust( top=0.95, bottom=0.16, left=0.10, right=0.95 )
plot( Load_1_plt, Eff_1_plt, 'r-' )
plot( Load_2_plt, Eff_2_plt, 'g-' )
xlabel( '$\\Lambda = \\dfrac{P_\\mathrm{turb}}{P_\\mathrm{rated}}$');
ylabel( '$\\eta_\\mathrm{gen}$')
savefig( 'GeneratorEff_plot.pdf' )
savefig( 'GeneratorEff_plot.svg' )
```

```python

```