# Cooling Fraction Function

- Using **scipy.interpolate.RectBivariateSpline** to interpolate function:
- Linear interpolation (kx=1, ky=1)

```python
import numpy as np
import pandas as pd
import sympy as s
import math
import glob

import matplotlib
import matplotlib.pyplot as plt
from cycler import cycler

from scipy import interpolate

from astropy import units as u
from astropy import constants as const
from astropy.units import imperial
imperial.enable()

%matplotlib inline
import os
#home_dir = os.environ['/Users/eriksolhaug'] + '/'
import pyCloudy as pc
```

```python
pd.set_option("display.max_rows", None, "display.max_columns", None)
pd.set_option('precision', 16)
pd.set_option('display.float_format', lambda x: '%.3e' % x)
```

```python
# The directory in which we will have the model
# You may want to change this to a different place so that the current directory
# will not receive all the Cloudy files.
dir_ = '/Users/eriksolhaug/cloudy/c17.02/cloudoutputs/'

user_dir = '/Users/eriksolhaug/cloudy/c17.02/'

# Define verbosity to high level (will print errors, warnings and messages)
pc.log_.level = 3
```

```python
def calccoolingfrac(model_name, model_dir):
    """
    Returns FUNCTION object f(X, Y) where X is a hydrogen density (LOG) and Y is a temperature (LOG). The output fraction is in LOG.

    The temperature range given to the Cloudy input file is 10^4 - 10^7 K
    and the hydrogen density range given to the Cloudy input file is 10^-5.0 - 10^-3.0 cm^-3

    Inputs:
    model_name - a string, name of model run in CLOUDY (f.ex. 'model_41')
    model_dir - a string, directory containing model output files

    Output:
    f - an object, function yielding cooling fraction for the requested for the given model in LOG as a function of hydrogen density and temperature
    To get the cooling fraction for a certain hden X and temperature Y, simply call f(X, Y) and take the exponent 10**f(X, Y) of this to find
    the cooling fraction for the model at the specified hydrogen density and temperature.

    An example:
                In[]:
                f = calccolfrac('model_47', '/Users/eriksolhaug/cloudy/c17.02/cloudoutputs/')
                10**f(-5.0, 5.5)
                Out[]:
                array([[4.50693322e-33]]) -- where 4.50693322e-33 is the cooling fraction of OVI for hydrogen density of 10**-5.0 and a temperature of 10**5.5 Kelvin
    """

    # Interpolating function for the data computed in Cloudy
    grid_df = pd.read_csv(f'{model_dir}/{model_name}.grid', sep='\t')
    hdengrid = grid_df.loc[:, 'HDEN=%f L']
    tempgrid = grid_df.loc[:, 'CONSTANT ']
    x = hdengrid
    y = tempgrid

    # Loading .cool file
    cols = ['#depth cm','Temp K','Htot erg/cm3/s','Ctot erg/cm3/s','cool fracs']
    cool_df = pd.read_csv(f'{model_dir}{model_name}.cool', sep = '\t', usecols=cols)

    cool_df = cool_df.iloc[:, 0:4] # Excluding cool fracs column which contains unnecessary info, we want the H... and C... columns

    # Getting fractional columns
    z_array = []
    for index in range(0, x.size):
        frac_cool = float(cool_df.iloc[index*2, -1]) # Need to convert this to float as values are reported as strings
        z_array.append(np.log10(frac_cool+1e-50))  #Adding a small value to avoid case of log(0)
    z = pd.DataFrame(z_array, columns=['z'])

    if model_name == 'model_43':
        step = 5
    elif model_name == 'model_45':
        step = 41
    elif model_name == 'model_46':
        step = 401
    elif model_name == 'model_47':
        step = 101
    else:
        step = 11

    # Putting vectors in dataframe representation
    xyz = pd.DataFrame({'x' : x, 'y' : y, 'z' : z['z']})

    # Simplifying x and y inputs
    xi = xyz['x'][:step]
    yi = xyz['y'][::step]

    # Preparing spline arrays
    twoDarray = []
    for i in range(len(xi)):
        array = []
        for j in range(len(yi)):
            idx = i + j*step
            array.append(xyz['z'][idx])
        twoDarray.append(array)

    # Simplifying z inputs
    zi = twoDarray

    print(xi, yi, zi)
    print(len(xi), len(yi), len(zi))

    # INTERPOLATION
    f = interpolate.RectBivariateSpline(xi, yi, zi, kx=1, ky=1) # Linear determined by kx, ky

    # Displaying match between old fractions and interpolated function
    interpolated_z = []
    for temp in yi:
        for hden in xi:
            interpolated_z.append(f(hden, temp))
    interpolated_z = np.concatenate(interpolated_z)

    print(interpolated_z)
    print(min(interpolated_z))

    return f
```

```python
def coolingplot(keyword, second_val_arr, model_name, model_dir, plot_dir):
    '''
    Function used to make and save plots of the fractional columns for different elements

    Input:
    keyword - a string, either 'temp' or 'hden' for what needs to be plotted against
    second_val - a float in LOG, value for either temp or hden (whatever not requested by keyword) - the plot needs a set temp or hden and this is set by this parameter
    model_name - a string, name of model run in CLOUDY (f.ex. 'model_42')
    model_dir - a string, directory containing model output files
    plot_dir - a string, directory for where to save plot
    '''

    # Plotting
    tick_fontsize = 16
    axis_fontsize = 22
    lwidth = 3
    lstyle = '-'

    fig, ax = plt.subplots(1, 1)
    fig.set_size_inches(8, 5)
    fig.tight_layout(w_pad = 10.0)

    if keyword == 'temp':
        constant_arr = second_val_arr
        vary = np.arange(4, 7, 0.01)
        xaxis = 10**vary
    elif keyword == 'hden':
        vary = np.arange(-5.0, -3.0, 0.01)
        constant_arr = second_val_arr
        xaxis = 10**vary
    else:
        print('Not a valid keyword. Needs either "temp" or "hden".')

    f = calccoolingfrac(model_name, model_dir)
    if keyword == 'temp':
        other_keyword = 'hden'
        for constant in constant_arr:
            ax.plot(xaxis, 10**f(constant, vary)[0], linewidth=lwidth, label=f'10^{constant}cm-3', linestyle=lstyle)
    elif keyword == 'hden':
        other_keyword = 'temp'
        for constant in constant_arr:
            ax.plot(xaxis, 10**f(vary, constant), linewidth=lwidth, label=f'10^{constant}K', linestyle=lstyle)

    ax.set_xscale('log')
    ax.set_yscale('log')
    if keyword == 'temp':
        ax.set_xlabel(r'Temperature ($K$)', fontsize = 12)
    else:
        ax.set_xlabel(r'Hydrogen density ($cm^{-3}$)', fontsize = 12)
    ax.set_ylabel('Cooling Fraction', fontsize = 12)
#     ax.set_ylim(1e-36, 1e-25)
    ax.set_title(f'Cooling fractions | Constant {other_keyword}', fontsize=18, fontweight='bold', pad=15)
    for tick in ax.xaxis.get_major_ticks():
        tick.label.set_fontsize(fontsize=tick_fontsize)
    for tick in ax.yaxis.get_major_ticks():
        tick.label.set_fontsize(fontsize=tick_fontsize)
    ax.xaxis.label.set_fontsize(axis_fontsize)
    ax.yaxis.label.set_fontsize(axis_fontsize)
    ax.tick_params(which='major', width=2, length=8)
    ax.tick_params(which='minor', width=1, length=5)
    ax.grid(linestyle='--')
    ax.legend(bbox_to_anchor=(1.05, 1), loc='upper left', prop={'size': 18})
    ax.set_prop_cycle(cycler('color', ['c', 'm', 'y', 'k']))

    fig.savefig(f'{plot_dir}{keyword}vary_coolingfractions.pdf', bbox_inches="tight")
```

### Example Executions:

```python
model_name = 'model_47'
model_dir = '/Users/eriksolhaug/cloudy/c17.02/cloudoutputs/'
f = calccoolingfrac(model_name, model_dir)
```

```python
10**f(-5.0,5.5)
```

$\: \uparrow$ This is the cooling fraction for the model at a given hydrogen density and temperature.

One can use the same procedure for other ions:

### Plotting fractional columns for all ions (C, N, O, Si)

```python
model_name = 'model_47'
plot_dir = '/Users/eriksolhaug/cloudy/c17.02/es/es_data/cooling_fractions/' + model_name + '/'
keyword_array = ['temp', 'hden']
temp_val_array = np.arange(-5.0, -2.5, 0.5)
hden_val_array = np.arange(4.0, 6.5, 0.5)
for keyword in keyword_array:
    if keyword == 'temp':
        second_val_array = temp_val_array
    elif keyword == 'hden':
        second_val_array = hden_val_array
    plot = coolingplot(keyword, second_val_array, model_name, model_dir, plot_dir)
```

```python

```

```python
# Individual plots:

# def coolingplot(keyword, second_val, model_name, model_dir, plot_dir):
#     '''
#     Function used to make and save plots of the fractional columns for different elements

#     Input:
#     keyword - a string, either 'temp' or 'hden' for what needs to be plotted against
#     second_val - a float in LOG, value for either temp or hden (whatever not requested by keyword) - the plot needs a set temp or hden and this is set by this parameter
#     model_name - a string, name of model run in CLOUDY (f.ex. 'model_42')
#     model_dir - a string, directory containing model output files
#     plot_dir - a string, directory for where to save plot
#     '''

#     # Plotting
#     tick_fontsize = 16
#     axis_fontsize = 22
#     lwidth = 3
#     lstyle = '-'

#     fig, ax = plt.subplots(1, 1)
#     fig.set_size_inches(8, 5)
#     fig.tight_layout(w_pad = 10.0)

#     if keyword == 'temp':
#         constant = second_val
#         vary = np.arange(4, 7, 0.01)
#         xaxis = 10**vary
#     elif keyword == 'hden':
#         vary = np.arange(-5.0, -3.0, 0.01)
#         constant = second_val
#         xaxis = 10**vary
#     else:
#         print('Not a valid keyword. Needs either "temp" or "hden".')

#     f = calccoolingfrac(model_name, model_dir)
#     if keyword == 'temp':
#         other_keyword = 'hden'
#         ax.plot(xaxis, 10**f(constant, vary)[0], linewidth=lwidth, label=f'{model_name}', linestyle=lstyle)
#     elif keyword == 'hden':
#         other_keyword = 'temp'
#         ax.plot(xaxis, 10**f(vary, constant), linewidth=lwidth, label=f'{model_name}', linestyle=lstyle)

#     ax.set_xscale('log')
#     ax.set_yscale('log')
#     if keyword == 'temp':
#         ax.set_xlabel(r'Temperature ($K$)', fontsize = 12)
#     else:
#         ax.set_xlabel(r'Hydrogen density ($cm^{-3}$)', fontsize = 12)
#     ax.set_ylabel('Cooling Fraction', fontsize = 12)
# #     ax.set_ylim(1e-36, 1e-25)
#     ax.set_title(f'Cooling fractions | Constant {other_keyword}: 10^{second_val}', fontsize=18, fontweight='bold', pad=15)
#     for tick in ax.xaxis.get_major_ticks():
#         tick.label.set_fontsize(fontsize=tick_fontsize)
#     for tick in ax.yaxis.get_major_ticks():
#         tick.label.set_fontsize(fontsize=tick_fontsize)
#     ax.xaxis.label.set_fontsize(axis_fontsize)
#     ax.yaxis.label.set_fontsize(axis_fontsize)
#     ax.tick_params(which='major', width=2, length=8)
#     ax.tick_params(which='minor', width=1, length=5)
#     ax.grid(linestyle='--')
#     ax.legend(bbox_to_anchor=(1.05, 1), loc='upper left', prop={'size': 18})
#     ax.set_prop_cycle(cycler('color', ['c', 'm', 'y', 'k']))

#     fig.savefig(f'{plot_dir}{keyword}vary_{second_val}_coolingfractions.pdf', bbox_inches="tight")
```


```
def check_goldbach_for_num(n, primes_set):
    for x in primes_set:
        for y in primes_set:
            if x + y == n :
                return True
    return False

```

```
from sympy.ntheory.generate import primerange
primes = set(primerange(50,1000))
check_goldbach_for_num(116, primes)
```


# Singly connected bonds

In this notebook we'll explore subsets of the spanning cluster and illustrate the _singly connected bonds_ of the spanning cluster. A singly connected bond is a collection of sites that connect two clusters by a single path. That is, if we remove a single site from this set we will create two separate clusters that are no longer connected.

```python
import warnings
import numpy as np
import tqdm
import matplotlib.pyplot as plt
import seaborn as sns
import scipy.ndimage
import sklearn.linear_model

from walker import walk
```

```python
sns.set(color_codes=True)
```

```python
# Shape of system
shape = (100, 100)
# Critical probability for percolation
p_c = 0.59275
# Probability of set site
p = p_c #0.586
# Number if trials to create a percolating cluster
num_attempts = 1000
```

We create a system which contains a percolating cluster. This is needed for the walker algorithm to work.

```python
for i in tqdm.tqdm_notebook(range(num_attempts)):
    z = np.random.rand(*shape) < p
    labels, num_features = scipy.ndimage.measurements.label(z)

    # Find label of percolating cluster. This is done by checking if a
    # label exists in both the uppermost and the lowermost rows of the
    # image.
    label_percolating = np.intersect1d(labels[0, :], labels[-1, :])

    # We ignore the infinite system , i.e., the label 0
    perc_label = label_percolating[label_percolating > 0]

    # If we find a spanning cluster, break the loop
    if len(perc_label) > 0:
        print(f"Percolating cluster found!")
        break
```

    Percolating cluster found!

Having found a system with a percolating cluster and extracted the label of the spanning cluster, we proceed to compute the mass of each cluster and fetch the sites contained in the spanning cluster.

```python
label_list = np.arange(num_features + 1)
area = scipy.ndimage.measurements.sum(z, labels, index=label_list)
area_image = area[labels]
spanning_cluster = labels == perc_label[0]
```

Below we plot all the connected clusters in the system and color them proportionally to their mass. That is, a high mass yields a bright color and vice versa.

```python
plt.figure(figsize=(14, 10))

plt.imshow(area_image, interpolation="nearest", origin="upper")
plt.colorbar()
plt.title("Plot of the clusters colored according to their area")
plt.show()
```

```python
print(f"Area of largest cluster: {np.max(area)}")
print(f"Area of second largest cluster: {np.sort(area)[-2]}")
```

    Area of largest cluster: 3193.0
    Area of second largest cluster: 722.0

We proceed to show a plot of the spanning cluster without any of the other clusters.

```python
plt.figure(figsize=(14, 10))

plt.imshow(spanning_cluster, interpolation="nearest", origin="upper")
plt.title("Plot of the spanning cluster")
plt.show()
```

Here we can see the sites contained in the spanning cluster visualized. Next we perform a left-right walk on the spanning cluster in order to find singly connected bonds, the backbone of the cluster and the dangling ends.

```python
left, right = walk(spanning_cluster)
```

```python
plt.figure(figsize=(14, 10))

plt.imshow(left, interpolation="nearest", origin="upper")
plt.colorbar()
plt.title("Plot of the path chosen by the left-walker")
plt.show()
```

In this image we see the path chosen by the left-walker. Bright colors signifies that a site is visited more than once.

```python
plt.figure(figsize=(14, 10))

plt.imshow(right, interpolation="nearest", origin="upper")
plt.colorbar()
plt.title("Plot of path chosen by the right-walker")
plt.show()
```

Note that path is found by starting from the top of the spanning cluster, i.e., at row `0`, and the walkers move downwards to higher row numbers.

```python
plt.figure(figsize=(14, 10))

plt.imshow((left * right) > 0, interpolation="nearest", origin="upper")
plt.show()
```

```python
plt.figure(figsize=(14, 10))

plt.imshow(left + right, interpolation="nearest", origin="upper")
plt.colorbar()
plt.show()
```

## Computing the mass of the singly connected bonds

We can find the set of sites in the singly connected bonds by multiplying the `right` and `left` arrays. A singly connected site is a site where both `right > 0` and `left > 0`.

```python
print(f"Mass SC: {np.sum((right * left) > 0)}")
```

    Mass SC: 23

We expect the mass of the singly connected sites, $M_{SC}$, to scale as
\begin{gather}
    M_{SC} \propto L^{D_{SC}}
    \implies
    \log(M_{SC}) = \log(C) + D_{SC} \log(L),
\end{gather}
where $D_{SC}$ is the _fractal dimension_ of the singly connected bonds and $C$ is a constant. To estimate $D_{SC}$ we compute the mass of the singly connected bonds for several system sizes $L$.

```python
def compute_mass_sc(L, num_samples, num_attempts=1000, p=p_c):
    mass_arr = np.zeros(num_samples)
    for i in tqdm.tqdm_notebook(range(num_samples)):
        for attempt in range(num_attempts):
            z = np.random.rand(L, L) < p
            labels, num_features = scipy.ndimage.measurements.label(z)
            label_percolating = np.intersect1d(labels[0, :], labels[-1, :])
            perc_label = label_percolating[label_percolating > 0]

            # If we find a spanning cluster, break the loop
            if len(perc_label) > 0:
                break

        else:
            warnings.warn("Percolating cluster not found!")

        spanning_cluster = labels == perc_label[0]
        left, right = walk(spanning_cluster)

        mass_arr[i] = np.sum((left * right) > 0)

    return mass_arr
```

```python
num_samples = 100
L_list = list(map(lambda x: x * 100, range(1, 11)))
mass_list = []
```

```python
for L in L_list:
    mass_arr = compute_mass_sc(L, num_samples)
    mass_list.append(np.mean(mass_arr))
```

```python
clf = sklearn.linear_model.LinearRegression().fit(
    np.log(L_list)[:, np.newaxis], np.log(mass_list)[:, np.newaxis]
)

D_sc = clf.coef_[0, 0]
log_C = clf.intercept_[0]

print(f"Singly-connected fractal dimension: {D_sc}")
```

    Singly-connected fractal dimension: 0.7630027311558977

The true value of the fractal dimension is given by
\begin{align}
    D_{SC} = \frac{3}{4} = 0.75,
\end{align}
which we see is quite close to the value we find.

```python
D_sc_true = 3 / 4
```

```python
plt.figure(figsize=(14, 10))

plt.loglog(L_list, mass_list, label="Measured")
plt.loglog(
    L_list,
    np.exp(log_C) * np.array(L_list) ** D_sc,
    "--",
    label=r"Estimated",
)
plt.legend(loc="best")
plt.title(r"Scaling of mass of singly connected bonds as a function of system size $L$")
plt.xlabel(r"$L$")
plt.ylabel(r"$M_{SC}$")
plt.show()
```

In this figure we can see how the mass of the singly connected bonds scales as a function of the system size $L$. Using linear regression we estimate the fractal dimension of the singly connected bonds as shown above.

We now proceed to find the behaviour of the density of the singly connected bonds by
\begin{align}
    P_{SC} = \frac{M_{SC}}{L^d},
\end{align}
where $d = 2$ is the dimension of the system. We find the density as a function of $p - p_c$.

```python
p_arr = np.linspace(p_c, 0.7, 11)
p_min_pc = p_arr - p_c

L_arr = np.array([100, 200, 500, 1000])
mass_sc = np.zeros((len(L_arr), len(p_arr)))

M = 100
```

```python
for i, L in enumerate(L_arr):
    for j, p in enumerate(p_arr):
        mass_sc[i, j] = np.mean(compute_mass_sc(L, M, p=p))
```

```python
plt.figure(figsize=(14, 10))

for i, L in enumerate(L_arr):
    plt.plot(
        p_min_pc,
        mass_sc[i] / L ** 2,
        label=fr"$L = {L}$",
    )

plt.legend(loc="best")
plt.title(r"Density of the singly connected bonds as a function of $p - p_c$ for various $L$")
plt.xlabel(r"$p - p_c$")
plt.ylabel(r"$P_{SC}$")
plt.show()
```

In the above figure we can see how the density decreases as the system size $L$ increases. We have only included values for $p \geq p_c$ as $p < p_c$ seldom contains a spanning cluster. For $p = p_c$ we on average get the smallest spanning cluster with the highest amount of singly connected bonds (confirmed from the plot), whereas a higher value of $p$ gives more dense spanning clusters. This means that the mass of the singly connected bonds sink. We can therefore expect to observe a trend where
\begin{align}
    P_{SC} \propto |p - p_c|^{-\gamma},
\end{align}
which is only applicable as $p \to p_c$. Fixing the system size $L$ and taking the logarithm on both sides we can estimate $-\gamma$ by linear regression.
\begin{align}
    \log\left( P_{SC} \right) = \log(C) - \gamma \log\left(| p - p_c | \right),
\end{align}
where $C$ is a constant.

```python
L = 500
M = 1000
p_arr = np.linspace(p_c, 0.8, 11)[1:]
p_min_pc = np.abs(p_arr - p_c)
P_arr = np.zeros_like(p_min_pc)
```

```python
for i, p in enumerate(p_arr):
    mass = np.mean(compute_mass_sc(L, M, p=p))
    P_arr[i] = mass / L ** 2
```

```python
clf = sklearn.linear_model.LinearRegression().fit(
    np.log(p_min_pc[:, np.newaxis]), np.log(P_arr[:, np.newaxis]),
)

gamma = -clf.coef_[0, 0]
log_C = clf.intercept_[0]

print(f"gamma = {gamma}")
```

    gamma = 1.0442599879955523

We see that $\gamma$ is very close to $1$.

```python
plt.figure(figsize=(14, 10))

plt.loglog(p_min_pc, P_arr, "o", label="Data points")
plt.loglog(
    p_min_pc,
    np.exp(log_C) * p_min_pc ** (-gamma),
    label="Approximation"
)

plt.legend(loc="best")
plt.title(fr"Scaling of density of singly connected bonds for $L = {L}$")
plt.xlabel(r"$|p - p_c|$")
plt.ylabel(r"$P_{SC}$")
plt.show()
```

In the figure above we can see how the density of the singly connected bonds follows a power law for $p \to p_c$.


# 线性模型的卷积神经网络特征

```python
# Rather than importing everything manually, we'll make things easy
#   and load them all in utils.py, and just import them from there.
%matplotlib inline
import imp
import utils
imp.reload(utils)
from utils import *
```

    C:\Anaconda3\lib\site-packages\h5py\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.
      from ._conv import register_converters as _register_converters
    Using TensorFlow backend.

## 介绍

我们需要找到一种方法，将imagenet的预测转换成猫或狗的二分概率，因为这是Kaggle竞赛要求我们提交的。我们可以使用imagenet层级来下载每个dog和cat组中的所有imagenet类别的列表，然后可以用各种方式解决我们的问题，例如:

- 找到最大的可能是猫或狗的概率，然后使用这个概率对应的标签值
- 计算所有猫类的平均概率，并与所有狗类的平均概率进行比较。

但这些方法也有一些缺点:

- 我们本应该数据中学到一些知识，而不是手动的硬编码。
- 忽略预测中可获得的资料;例如，如果模型预测图像中有一根骨头，它更可能是狗而不是猫。

这两个问题的一个非常简单的解决方案是学习一个线性模型，该模型使用来自imagenet模型的1000个预测作为输入，而dog/cat标签作为目标。

```python
%matplotlib inline
from __future__ import division,print_function
import os, json
from glob import glob
import numpy as np
import scipy
from sklearn.preprocessing import OneHotEncoder
from sklearn.metrics import confusion_matrix
np.set_printoptions(precision=4, linewidth=100)
from matplotlib import pyplot as plt
import utils
imp.reload(utils)
from utils import plots, get_batches, plot_confusion_matrix, get_data
```

```python
from numpy.random import random, permutation
from scipy import misc, ndimage
from scipy.ndimage.interpolation import zoom

import keras
from keras import backend as K
from keras.utils.data_utils import get_file
from keras.models import Sequential
from keras.layers import Input,Dense,Activation,ZeroPadding2D,Dropout,Conv2D,MaxPool2D,Flatten,Lambda,BatchNormalization
from keras.optimizers import SGD, RMSprop
from keras.preprocessing import image
np.random.seed(42)
```

## Keras中的线性模型

It turns out that each of the Dense() layers is just a *linear model*, followed by a simple *activation function*. We'll learn about the activation function later - first, let's review how linear models work.

每一个全连接层原来就是一个线性模型，

A linear model is (as I'm sure you know) simply a model where each row is calculated as *sum(row * weights)*, where *weights* needs to be learnt from the data, and will be the same for every row. For example, let's create some data that we know is linearly related:

```python
x = random((30,2))
x.shape
```

    (30, 2)

```python
x[:5]
```

    array([[0.3745, 0.9507],
           [0.732 , 0.5987],
           [0.156 , 0.156 ],
           [0.0581, 0.8662],
           [0.6011, 0.7081]])

```python
x = random((30,2))
y = np.dot(x, [2., 3.]) + 1.  # 直接定义参数[2,3] 1
```

```python
x[:5]
```

    array([[0.3887, 0.2713],
           [0.8287, 0.3568],
           [0.2809, 0.5427],
           [0.1409, 0.8022],
           [0.0746, 0.9869]])

```python
type(y)
```

    numpy.ndarray

```python
y.shape
```

    (30,)

```python
y
```

    array([2.5914, 3.7277, 3.19  , 3.6884, 4.1098, 3.1406, 3.4574, 4.6007, 2.7647, 2.0645, 4.5961,
           1.8525, 2.5975, 4.3719, 4.1911, 3.3789, 4.2054, 4.0233, 3.3281, 1.3745, 2.9721, 3.1544,
           3.563 , 4.0874, 1.6885, 2.0632, 5.2838, 4.8812, 3.1671, 4.4031])

We can use keras to create a simple linear model (*Dense()* - with no activation - in Keras) and optimize it using SGD to minimize mean squared error (*mse*):

建立一个Keras的顺序模型，不适用激活函数，这意味着这是一个线性模型，使用`SGD`优化算法

```python
lm = Sequential([Dense(1,input_shape=[2,])])
lm.compile(optimizer=SGD(lr=0.01),loss='mse')
lm.evaluate(x,y,verbose=1)
```

    30/30 [==============================] - 1s 35ms/step

    17.28533935546875

上面没有经过训练，直接评估，看到`MSE`误差很大

```python
# 训练
lm.fit(x,y,epochs=200,batch_size=1,verbose=0)
```

    <keras.callbacks.History at 0x2748cda0f28>

```python
lm.evaluate(x, y, verbose=0)
```

    2.7106448214908596e-06

最后看看训练后的权重，看到很接近2.0，3.0，1.0

```python
lm.get_weights()
```

    [array([[1.9969],
            [2.9948]], dtype=float32), array([1.0042], dtype=float32)]

## 用vgg16的预测值训练线形模型

Using a Dense() layer in this way, we can easily convert the 1,000 predictions given by our model into a probability of dog vs cat--simply train a linear model to take the 1,000 predictions as input, and return dog or cat as output, learning from the Kaggle data. This should be easier and more accurate than manually creating a map from imagenet categories to one dog/cat category.

### 训练模型

We start with some basic config steps. We copy a small amount of our data into a 'sample' directory, with the exact same structure as our 'train' directory--this is *always* a good idea in *all* machine learning, since we should do all of our initial testing using a dataset small enough that we never have to wait for it.

```python
path = "data/dogscats/"
# path = "data/dogscats/"
model_path = "data/dogscats/models/"
if not os.path.exists(model_path): os.mkdir(model_path)
```

We will process as many images at a time as our graphics card allows. This is a case of trial and error to find the max batch size - the largest size that doesn't give an out of memory error.

```python
batch_size=16
```

引入之前定义好的VGG16模型:

```python
from myvgg16 import Vgg16
#imp.reload(myvgg16)
vgg = Vgg16()
model = vgg.model # 调用自己定义的原始Vgg16模型，而不是finetune模型
```

```python
model.summary()
```

    _________________________________________________________________
    Layer (type)                 Output Shape              Param #
    =================================================================
    lambda_1 (Lambda)            (None, 3, 224, 224)       0
    _________________________________________________________________
    zero_padding2d_1 (ZeroPaddin (None, 3, 226, 226)       0
    _________________________________________________________________
    conv2d_1 (Conv2D)            (None, 64, 224, 224)      1792
    _________________________________________________________________
    zero_padding2d_2 (ZeroPaddin (None, 64, 226, 226)      0
    _________________________________________________________________
    conv2d_2 (Conv2D)            (None, 64, 224, 224)      36928
    _________________________________________________________________
    max_pooling2d_1 (MaxPooling2 (None, 64, 112, 112)      0
    _________________________________________________________________
    zero_padding2d_3 (ZeroPaddin (None, 64, 114, 114)      0
    _________________________________________________________________
    conv2d_3 (Conv2D)            (None, 128, 112, 112)     73856
    _________________________________________________________________
    zero_padding2d_4 (ZeroPaddin (None, 128, 114, 114)     0
    _________________________________________________________________
    conv2d_4 (Conv2D)            (None, 128, 112, 112)     147584
    _________________________________________________________________
    max_pooling2d_2 (MaxPooling2 (None, 128, 56, 56)       0
    _________________________________________________________________
    zero_padding2d_5 (ZeroPaddin (None, 128, 58, 58)       0
    _________________________________________________________________
    conv2d_5 (Conv2D)            (None, 256, 56, 56)       295168
    _________________________________________________________________
    zero_padding2d_6 (ZeroPaddin (None, 256, 58, 58)       0
    _________________________________________________________________
    conv2d_6 (Conv2D)            (None, 256, 56, 56)       590080
    _________________________________________________________________
    zero_padding2d_7 (ZeroPaddin (None, 256, 58, 58)       0
    _________________________________________________________________
    conv2d_7 (Conv2D)            (None, 256, 56, 56)       590080
    _________________________________________________________________
    max_pooling2d_3 (MaxPooling2 (None, 256, 28, 28)       0
    _________________________________________________________________
    zero_padding2d_8 (ZeroPaddin (None, 256, 30, 30)       0
    _________________________________________________________________
    conv2d_8 (Conv2D)            (None, 512, 28, 28)       1180160
    _________________________________________________________________
    zero_padding2d_9 (ZeroPaddin (None, 512, 30, 30)       0
    _________________________________________________________________
    conv2d_9 (Conv2D)            (None, 512, 28, 28)       2359808
    _________________________________________________________________
    zero_padding2d_10 (ZeroPaddi (None, 512, 30, 30)       0
    _________________________________________________________________
    conv2d_10 (Conv2D)           (None, 512, 28, 28)       2359808
    _________________________________________________________________
    max_pooling2d_4 (MaxPooling2 (None, 512, 14, 14)       0
    _________________________________________________________________
    zero_padding2d_11 (ZeroPaddi (None, 512, 16, 16)       0
    _________________________________________________________________
    conv2d_11 (Conv2D)           (None, 512, 14, 14)       2359808
    _________________________________________________________________
    zero_padding2d_12 (ZeroPaddi (None, 512, 16, 16)       0
    _________________________________________________________________
    conv2d_12 (Conv2D)           (None, 512, 14, 14)       2359808
    _________________________________________________________________
    zero_padding2d_13 (ZeroPaddi (None, 512, 16, 16)       0
    _________________________________________________________________
    conv2d_13 (Conv2D)           (None, 512, 14, 14)       2359808
    _________________________________________________________________
    max_pooling2d_5 (MaxPooling2 (None, 512, 7, 7)         0
    _________________________________________________________________
    flatten_1 (Flatten)          (None, 25088)             0
    _________________________________________________________________
    dense_2 (Dense)              (None, 4096)              102764544
    _________________________________________________________________
    dropout_1 (Dropout)          (None, 4096)              0
    _________________________________________________________________
    dense_3 (Dense)              (None, 4096)              16781312
    _________________________________________________________________
    dropout_2 (Dropout)          (None, 4096)              0
    _________________________________________________________________
    dense_4 (Dense)              (None, 1000)              4097000
    =================================================================
    Total params: 138,357,544
    Trainable params: 138,357,544
    Non-trainable params: 0
    _________________________________________________________________

Our overall approach here will be:
我们这里的总体方法是：

1. 获得每个图片的真实标签值。
2. 获得每个图片属于1000个类型的概率值。
3. 把这些预测简单的作为线性模型的输入。

首先，要获取训练集和验证集：

```python
# Use batch size of 1 since we're just doing preprocessing on the CPU
val_batches = get_batches(path+'valid', shuffle=False, batch_size=1)
batches = get_batches(path+'train', shuffle=False, batch_size=1)
```

    Found 2000 images belonging to 2 classes.
    Found 23000 images belonging to 2 classes.

每次使用数据的时候重复的读取，变化不是必须的。通过`bcolz`函数我们能够压缩数组，存放到硬盘，`utils.py`提供了很有用的函数

```python
import bcolz
def save_array(fname, arr):
    c=bcolz.carray(arr, rootdir=fname, mode='w')
    c.flush()
def load_array(fname):
    return bcolz.open(fname)[:]
```

We have provided a simple function that joins the arrays from all the batches - let's use this to grab the training and validation data:

```python
val_data = get_data(path+'valid')
```

    Found 2000 images belonging to 2 classes.

```python
trn_data = get_data(path+'train')
```

    Found 23000 images belonging to 2 classes.

```python
trn_data.shape
```

    (23000, 3, 224, 224)

```python
save_array(model_path+'train_data.bc', trn_data)
save_array(model_path+'valid_data.bc', val_data)
```

通过`load_array()`读取：

```python
trn_data = load_array(model_path+'train_data.bc')
val_data = load_array(model_path+'valid_data.bc')
```

```python
trn_data[0]
```

    array([[[203., 206., 209., ..., 245., 241., 239.],
            [203., 206., 209., ..., 245., 242., 240.],
            [203., 206., 209., ..., 245., 243., 241.],
            ...,
            [154., 155., 156., ...,   3.,   3.,   3.],
            [153., 153., 154., ...,   2.,   2.,   2.],
            [151., 152., 153., ...,   1.,   1.,   1.]],

           [[164., 167., 170., ..., 203., 202., 200.],
            [164., 167., 170., ..., 205., 203., 201.],
            [164., 167., 170., ..., 204., 204., 202.],
            ...,
            [123., 124., 125., ...,   3.,   3.,   3.],
            [122., 122., 123., ...,   2.,   2.,   2.],
            [120., 121., 122., ...,   1.,   1.,   1.]],

           [[ 87.,  90.,  93., ..., 119., 123., 121.],
            [ 87.,  90.,  93., ..., 120., 124., 122.],
            [ 87.,  90.,  93., ..., 122., 125., 123.],
            ...,
            [ 56.,  57.,  58., ...,   1.,   1.,   1.],
            [ 55.,  55.,  56., ...,   0.,   0.,   0.],
            [ 53.,  54.,  55., ...,   0.,   0.,   0.]]], dtype=float32)

```python
val_data.shape
```

    (2000, 3, 224, 224)

Keras returns *classes* as a single column, so we convert to one hot encoding

Keras 返回的*classes* 是一维向量，所以我们要转换成独热编码

```python
def onehot(x):
    return np.array(OneHotEncoder().fit_transform(x.reshape(-1,1)).todense())
```

```python
val_classes = val_batches.classes # 原始的一维向量的类别
trn_classes = batches.classes
val_labels = onehot(val_classes)
trn_labels = onehot(trn_classes) # onehot编码的
```

```python
print("val_classes:",val_classes)
print("trn_classes:",trn_classes)
```

    val_classes: [0 0 0 ... 1 1 1]
    trn_classes: [0 0 0 ... 1 1 1]

```python
# 查看一个编码 ，这是一个onehot编码
val_labels[0]
```

    array([1., 0.])

```python
trn_labels.shape
```

    (23000, 2)

```python
trn_labels[:4]
```

    array([[1., 0.],
           [1., 0.],
           [1., 0.],
           [1., 0.]])

...and their 1,000 imagenet probabilties from VGG16--these will be the *features* for our linear model:

每个图片的1000种类别的概率作为我们线性模型的`特征值`

```python
model.summary()
```

    _________________________________________________________________
    Layer (type)                 Output Shape              Param #
    =================================================================
    lambda_1 (Lambda)            (None, 3, 224, 224)       0
    _________________________________________________________________
    zero_padding2d_1 (ZeroPaddin (None, 3, 226, 226)       0
    _________________________________________________________________
    conv2d_1 (Conv2D)            (None, 64, 224, 224)      1792
    _________________________________________________________________
    zero_padding2d_2 (ZeroPaddin (None, 64, 226, 226)      0
    _________________________________________________________________
    conv2d_2 (Conv2D)            (None, 64, 224, 224)      36928
    _________________________________________________________________
    max_pooling2d_1 (MaxPooling2 (None, 64, 112, 112)      0
    _________________________________________________________________
    zero_padding2d_3 (ZeroPaddin (None, 64, 114, 114)      0
    _________________________________________________________________
    conv2d_3 (Conv2D)            (None, 128, 112, 112)     73856
    _________________________________________________________________
    zero_padding2d_4 (ZeroPaddin (None, 128, 114, 114)     0
    _________________________________________________________________
    conv2d_4 (Conv2D)            (None, 128, 112, 112)     147584
    _________________________________________________________________
    max_pooling2d_2 (MaxPooling2 (None, 128, 56, 56)       0
    _________________________________________________________________
    zero_padding2d_5 (ZeroPaddin (None, 128, 58, 58)       0
    _________________________________________________________________
    conv2d_5 (Conv2D)            (None, 256, 56, 56)       295168
    _________________________________________________________________
    zero_padding2d_6 (ZeroPaddin (None, 256, 58, 58)       0
    _________________________________________________________________
    conv2d_6 (Conv2D)            (None, 256, 56, 56)       590080
    _________________________________________________________________
    zero_padding2d_7 (ZeroPaddin (None, 256, 58, 58)       0
    _________________________________________________________________
    conv2d_7 (Conv2D)            (None, 256, 56, 56)       590080
    _________________________________________________________________
    max_pooling2d_3 (MaxPooling2 (None, 256, 28, 28)       0
    _________________________________________________________________
    zero_padding2d_8 (ZeroPaddin (None, 256, 30, 30)       0
    _________________________________________________________________
    conv2d_8 (Conv2D)            (None, 512, 28, 28)       1180160
    _________________________________________________________________
    zero_padding2d_9 (ZeroPaddin (None, 512, 30, 30)       0
    _________________________________________________________________
    conv2d_9 (Conv2D)            (None, 512, 28, 28)       2359808
    _________________________________________________________________
    zero_padding2d_10 (ZeroPaddi (None, 512, 30, 30)       0
    _________________________________________________________________
    conv2d_10 (Conv2D)           (None, 512, 28, 28)       2359808
    _________________________________________________________________
    max_pooling2d_4 (MaxPooling2 (None, 512, 14, 14)       0
    _________________________________________________________________
    zero_padding2d_11 (ZeroPaddi (None, 512, 16, 16)       0
    _________________________________________________________________
    conv2d_11 (Conv2D)           (None, 512, 14, 14)       2359808
    _________________________________________________________________
    zero_padding2d_12 (ZeroPaddi (None, 512, 16, 16)       0
    _________________________________________________________________
    conv2d_12 (Conv2D)           (None, 512, 14, 14)       2359808
    _________________________________________________________________
    zero_padding2d_13 (ZeroPaddi (None, 512, 16, 16)       0
    _________________________________________________________________
    conv2d_13 (Conv2D)           (None, 512, 14, 14)       2359808
    _________________________________________________________________
    max_pooling2d_5 (MaxPooling2 (None, 512, 7, 7)         0
    _________________________________________________________________
    flatten_1 (Flatten)          (None, 25088)             0
    _________________________________________________________________
    dense_2 (Dense)              (None, 4096)              102764544
    _________________________________________________________________
    dropout_1 (Dropout)          (None, 4096)              0
    _________________________________________________________________
    dense_3 (Dense)              (None, 4096)              16781312
    _________________________________________________________________
    dropout_2 (Dropout)          (None, 4096)              0
    _________________________________________________________________
    dense_4 (Dense)              (None, 1000)              4097000
    =================================================================
    Total params: 138,357,544
    Trainable params: 138,357,544
    Non-trainable params: 0
    _________________________________________________________________

```python
trn_features = model.predict(trn_data, batch_size=batch_size)
val_features = model.predict(val_data, batch_size=batch_size)
```

```python
trn_features.shape
```

    (23000, 1000)

```python
save_array(model_path+'train_lastlayer_features.bc', trn_features)
save_array(model_path+'valid_lastlayer_features.bc', val_features)
```

We can load our training and validation features later without recalculating them:

```python
trn_features = load_array(model_path+'train_lastlayer_features.bc')
val_features = load_array(model_path+'valid_lastlayer_features.bc')
```

定义线性模型：

```python
lm = Sequential([Dense(2,activation='softmax',input_shape=(1000,))])
lm.compile(optimizer=RMSprop(lr=0.1),loss='categorical_crossentropy',metrics=['accuracy'])
```

We're ready to fit the model!

```python
batch_size=16
```

```python
lm.fit(trn_features,trn_labels,epochs=10,batch_size=batch_size,validation_data=(val_features,val_labels))
```

    Train on 23000 samples, validate on 2000 samples
    Epoch 1/10
    23000/23000 [==============================] - 4s 183us/step - loss: 0.3763 - acc: 0.8594 - val_loss: 0.4012 - val_acc: 0.8645
    Epoch 2/10
    23000/23000 [==============================] - 4s 158us/step - loss: 0.4121 - acc: 0.8656 - val_loss: 0.4421 - val_acc: 0.8670
    Epoch 3/10
    23000/23000 [==============================] - 4s 161us/step - loss: 0.4291 - acc: 0.8704 - val_loss: 0.4911 - val_acc: 0.8660
    Epoch 4/10
    23000/23000 [==============================] - 4s 161us/step - loss: 0.4534 - acc: 0.8709 - val_loss: 0.5135 - val_acc: 0.8670
    Epoch 5/10
    23000/23000 [==============================] - 4s 159us/step - loss: 0.4718 - acc: 0.8720 - val_loss: 0.5298 - val_acc: 0.8660
    Epoch 6/10
    23000/23000 [==============================] - 4s 159us/step - loss: 0.4884 - acc: 0.8720 - val_loss: 0.5478 - val_acc: 0.8720
    Epoch 7/10
    23000/23000 [==============================] - 4s 159us/step - loss: 0.5007 - acc: 0.8746 - val_loss: 0.5628 - val_acc: 0.8720
    Epoch 8/10
    23000/23000 [==============================] - 4s 158us/step - loss: 0.5125 - acc: 0.8726 - val_loss: 0.5803 - val_acc: 0.8620
    Epoch 9/10
    23000/23000 [==============================] - 4s 159us/step - loss: 0.5231 - acc: 0.8727 - val_loss: 0.6055 - val_acc: 0.8620
    Epoch 10/10
    23000/23000 [==============================] - 4s 157us/step - loss: 0.5295 - acc: 0.8749 - val_loss: 0.6079 - val_acc: 0.8655

    <keras.callbacks.History at 0x275a4cc22e8>

```python
lm.summary()
```

    _________________________________________________________________
    Layer (type)                 Output Shape              Param #
    =================================================================
    dense_5 (Dense)              (None, 2)                 2002
    =================================================================
    Total params: 2,002
    Trainable params: 2,002
    Non-trainable params: 0
    _________________________________________________________________

### 展示模型预测

Keras' *fit()* function conveniently shows us the value of the loss function, and the accuracy, after every epoch ("*epoch*" refers to one full run through all training examples). The most important metrics for us to look at are for the validation set, since we want to check for over-fitting.

- **Tip**: with our first model we should try to overfit before we start worrying about how to handle that - there's no point even thinking about regularization, data augmentation, etc if you're still under-fitting! (We'll be looking at these techniques shortly).

As well as looking at the overall metrics, it's also a good idea to look at examples of each of:
1. A few correct labels at random 随机的分类正确的
2. A few incorrect labels at random 随机的分类错误的
3. The most correct labels of each class (ie those with highest probability that are correct)
4. The most incorrect labels of each class (ie those with highest probability that are incorrect)
5. The most uncertain labels (ie those with probability closest to 0.5).

Let's see what we, if anything, we can from these (in general, these are particularly useful for debugging problems in the model; since this model is so simple, there may not be too much to learn at this stage.)

在验证集上预测，这样我们能找到正确和不正确的样本：

```python
val_features
```

    array([[6.8475e-06, 1.6689e-06, 6.1381e-07, ..., 4.1279e-05, 1.2564e-03, 1.2930e-05],
           [2.1105e-06, 3.7982e-05, 4.6157e-06, ..., 1.1608e-05, 2.7110e-04, 1.5983e-04],
           [9.9811e-07, 1.2790e-05, 5.2557e-07, ..., 2.0450e-06, 8.3339e-06, 1.4718e-04],
           ...,
           [1.2637e-06, 3.0298e-07, 1.1776e-06, ..., 1.0079e-06, 8.2599e-06, 3.5690e-04],
           [1.3638e-07, 1.3226e-08, 5.1778e-05, ..., 2.0121e-07, 6.7783e-06, 3.1354e-05],
           [4.4331e-06, 2.9501e-06, 5.6086e-06, ..., 9.5832e-07, 4.7147e-05, 3.0456e-05]],
          dtype=float32)

```python
preds = lm.predict(val_features,batch_size=batch_size) # 预测概率，而不是类别值
```

```python
preds
```

    array([[2.8740e-04, 9.9971e-01],
           [9.9477e-01, 5.2264e-03],
           [1.0000e+00, 1.2498e-09],
           ...,
           [1.3690e-02, 9.8631e-01],
           [1.1152e-01, 8.8848e-01],
           [6.7649e-02, 9.3235e-01]], dtype=float32)

```python
np.argmax(preds,axis=1)
```

    array([1, 0, 0, ..., 1, 1, 1], dtype=int64)

```python
preds = lm.predict_classes(val_features, batch_size=batch_size) # 直接预测是类别值，之前定义好的0，1
```

```python
preds # 是不是狗的标签
```

    array([1, 0, 0, ..., 1, 1, 1], dtype=int64)

```python
probs = lm.predict_proba(val_features, batch_size=batch_size)
```

```python
probs
```

    array([[2.8740e-04, 9.9971e-01],
           [9.9477e-01, 5.2264e-03],
           [1.0000e+00, 1.2498e-09],
           ...,
           [1.3690e-02, 9.8631e-01],
           [1.1152e-01, 8.8848e-01],
           [6.7649e-02, 9.3235e-01]], dtype=float32)

**这个看到这个函数其实跟`predict`是一样的**

Get the filenames for the validation set, so we can view images:

Keras的batch有很多有用的方法：

```python
val_batches.class_indices # 获得类别索引说明
```

    {'cats': 0, 'dogs': 1}

```python
filenames = val_batches.filenames
```

```python
# Number of images to view for each visualization task
n_view = 4
```

根据索引画出验证集的图片：

```python
def plots_idx(idx, titles=None):
    plots([image.load_img(path + 'valid/' + filenames[i]) for i in idx], titles=titles)
```

```python
#1. 随机的分类正确的，

correct = np.where(preds ==val_labels[:,1])[0] # 所有分类正确的索引
idx = permutation(correct)[:n_view]
plots_idx(idx,probs[idx])
```

```python
probs[idx]
```

    array([[0.0096, 0.9904],
           [0.0019, 0.9981],
           [0.9708, 0.0292],
           [0.5387, 0.4613]], dtype=float32)

```python
#2. 随机的分类不正确的
incorrect = np.where(preds!=val_labels[:,1])[0]
idx = permutation(incorrect)[:n_view]
plots_idx(idx, probs[idx])
```

```python
#3 分类概率最高的是猫，实际上也是猫
correct_cats = np.where((preds==0)&(preds==val_labels[:,1]))[0]

probs[correct_cats] # 所有的分类是猫，切争取的概率值
```

    array([[9.9477e-01, 5.2264e-03],
           [1.0000e+00, 1.2498e-09],
           [9.8561e-01, 1.4393e-02],
           ...,
           [8.5192e-01, 1.4808e-01],
           [1.0000e+00, 8.0086e-09],
           [8.7883e-01, 1.2117e-01]], dtype=float32)

```python
# a[::-1] # 省略范围的开始下标和结束下标，步长为-1，整个数组头尾颠倒
most_correct_cats = np.argsort(probs[correct_cats])[::-1][:n_view]
most_correct_cats
```

    array([[1, 0],
           [1, 0],
           [1, 0],
           [1, 0]], dtype=int64)

```python
correct_cats[most_correct_cats]
```

    array([[2, 1],
           [2, 1],
           [2, 1],
           [2, 1]], dtype=int64)

```python
probs[correct_cats][most_correct_cats]
```

    array([[[1.0000e+00, 1.2498e-09],
            [9.9477e-01, 5.2264e-03]],

           [[1.0000e+00, 1.2498e-09],
            [9.9477e-01, 5.2264e-03]],

           [[1.0000e+00, 1.2498e-09],
            [9.9477e-01, 5.2264e-03]],

           [[1.0000e+00, 1.2498e-09],
            [9.9477e-01, 5.2264e-03]]], dtype=float32)

```python
#3. The images we most confident were cats, and are actually cats
correct_cats = np.where((preds==0) & (preds==val_labels[:,1]))[0]
most_correct_cats = np.argsort(probs[correct_cats])[::-1][:n_view]
plots_idx(correct_cats[most_correct_cats], probs[correct_cats][most_correct_cats])
```

```python
# as above, but dogs
correct_dogs = np.where((preds==1) & (preds==val_labels[:,1]))[0]
most_correct_dogs = np.argsort(probs[correct_dogs])[:n_view]
plots_idx(correct_dogs[most_correct_dogs], 1-probs[correct_dogs][most_correct_dogs])
```

```python
#3. The images we were most confident were cats, but are actually dogs
incorrect_cats = np.where((preds==0) & (preds!=val_labels[:,1]))[0]
most_incorrect_cats = np.argsort(probs[incorrect_cats])[::-1][:n_view]
if len(most_incorrect_cats):
    plots_idx(incorrect_cats[most_incorrect_cats], probs[incorrect_cats][most_incorrect_cats])
else:
    print('No incorrect cats!')
```

```python
#3. The images we were most confident were dogs, but are actually cats
incorrect_dogs = np.where((preds==1) & (preds!=val_labels[:,1]))[0]
most_incorrect_dogs = np.argsort(probs[incorrect_dogs])[:n_view]
if len(most_incorrect_dogs):
    plots_idx(incorrect_dogs[most_incorrect_dogs], 1-probs[incorrect_dogs][most_incorrect_dogs])
else:
    print('No incorrect dogs!')
```

```python
#5. The most uncertain labels (ie those with probability closest to 0.5).
most_uncertain = np.argsort(np.abs(probs-0.5))
plots_idx(most_uncertain[:n_view], probs[most_uncertain])
```

对于2分类任务来说，展示分类效果的应该是混淆矩阵最好：`Scikit-learn`提供一个方便函数

```python
cm = confusion_matrix(val_classes, preds)
```

We can just print out the confusion matrix, or we can show a graphical view (which is mainly useful for dependents with a larger number of categories).

```python
plot_confusion_matrix(cm, val_batches.class_indices)
```

### 关于激活函数

Do you remember how we defined our linear model? Here it is again for reference:

```python
lm = Sequential([ Dense(2, activation='softmax', input_shape=(1000,)) ])
```

And do you remember the definition of a fully connected layer in the original VGG?:

```python
model.add(Dense(4096, activation='relu'))
```

You might we wondering, what's going on with that *activation* parameter? Adding an 'activation' parameter to a layer in Keras causes an additional function to be called after the layer is calculated. You'll recall that we had no such parameter in our most basic linear model at the start of this lesson - that's because a simple linear model has no *activation function*. But nearly all deep model layers have an activation function - specifically, a *non-linear* activation function, such as tanh, sigmoid (```1/(1+exp(x))```), or relu (```max(0,x)```, called the *rectified linear* function). Why?

The reason for this is that if you stack purely linear layers on top of each other, then you just end up with a linear layer! For instance, if your first layer was ```2*x```, and your second was ```-2*x```, then the combination is: ```-2*(2*x) = -4*x```. If that's all we were able to do with deep learning, it wouldn't be very deep! But what if we added a relu activation after our first layer? Then the combination would be: ```-2 * max(0, 2*x)```. As you can see, that does not simplify to just a linear function like the previous example--and indeed we can stack as many of these on top of each other as we wish, to create arbitrarily complex functions.

And why would we want to do that? Because it turns out that such a stack of linear functions and non-linear activations can approximate any other function just as close as we want. So we can **use it to model anything**! This extraordinary insight is known as the *universal approximation theorem*. For a visual understanding of how and why this works, I strongly recommend you read Michael Nielsen's [excellent interactive visual tutorial](http://neuralnetworksanddeeplearning.com/chap4.html).

The last layer generally needs a different activation function to the other layers--because we want to encourage the last layer's output to be of an appropriate form for our particular problem. For instance, if our output is a one hot encoded categorical variable, we want our final layer's activations to add to one (so they can be treated as probabilities) and to have generally a single activation much higher than the rest (since with one hot encoding we have just a single 'one', and all other target outputs are zero). Our classication problems will always have this form, so we will introduce the activation function that has these properties: the *softmax* function. Softmax is defined as (for the i'th output activation): ```exp(x[i]) / sum(exp(x))```.

I suggest you try playing with that function in a spreadsheet to get a sense of how it behaves.

We will see other activation functions later in this course - but relu (and minor variations) for intermediate layers and softmax for output layers will be by far the most common.

# 更改模型

## 重新训练线形模型的最后一层

Since the original VGG16 network's last layer is Dense (i.e. a linear model) it seems a little odd that we are adding an additional linear model on top of it. This is especially true since the last layer had a softmax activation, which is an odd choice for an intermediate layer--and by adding an extra layer on top of it, we have made it an intermediate layer. What if we just removed the original final layer and replaced it with one that we train for the purpose of distinguishing cats and dogs? It turns out that this is a good idea - as we'll see!

We start by removing the last layer, and telling Keras that we want to fix the weights in all the other layers (since we aren't looking to learn new parameters for those other layers).

```python
vgg.model.summary()
```

    _________________________________________________________________
    Layer (type)                 Output Shape              Param #
    =================================================================
    lambda_1 (Lambda)            (None, 3, 224, 224)       0
    _________________________________________________________________
    zero_padding2d_1 (ZeroPaddin (None, 3, 226, 226)       0
    _________________________________________________________________
    conv2d_1 (Conv2D)            (None, 64, 224, 224)      1792
    _________________________________________________________________
    zero_padding2d_2 (ZeroPaddin (None, 64, 226, 226)      0
    _________________________________________________________________
    conv2d_2 (Conv2D)            (None, 64, 224, 224)      36928
    _________________________________________________________________
    max_pooling2d_1 (MaxPooling2 (None, 64, 112, 112)      0
    _________________________________________________________________
    zero_padding2d_3 (ZeroPaddin (None, 64, 114, 114)      0
    _________________________________________________________________
    conv2d_3 (Conv2D)            (None, 128, 112, 112)     73856
    _________________________________________________________________
    zero_padding2d_4 (ZeroPaddin (None, 128, 114, 114)     0
    _________________________________________________________________
    conv2d_4 (Conv2D)            (None, 128, 112, 112)     147584
    _________________________________________________________________
    max_pooling2d_2 (MaxPooling2 (None, 128, 56, 56)       0
    _________________________________________________________________
    zero_padding2d_5 (ZeroPaddin (None, 128, 58, 58)       0
    _________________________________________________________________
    conv2d_5 (Conv2D)            (None, 256, 56, 56)       295168
    _________________________________________________________________
    zero_padding2d_6 (ZeroPaddin (None, 256, 58, 58)       0
    _________________________________________________________________
    conv2d_6 (Conv2D)            (None, 256, 56, 56)       590080
    _________________________________________________________________
    zero_padding2d_7 (ZeroPaddin (None, 256, 58, 58)       0
    _________________________________________________________________
    conv2d_7 (Conv2D)            (None, 256, 56, 56)       590080
    _________________________________________________________________
    max_pooling2d_3 (MaxPooling2 (None, 256, 28, 28)       0
    _________________________________________________________________
    zero_padding2d_8 (ZeroPaddin (None, 256, 30, 30)       0
    _________________________________________________________________
    conv2d_8 (Conv2D)            (None, 512, 28, 28)       1180160
    _________________________________________________________________
    zero_padding2d_9 (ZeroPaddin (None, 512, 30, 30)       0
    _________________________________________________________________
    conv2d_9 (Conv2D)            (None, 512, 28, 28)       2359808
    _________________________________________________________________
    zero_padding2d_10 (ZeroPaddi (None, 512, 30, 30)       0
    _________________________________________________________________
    conv2d_10 (Conv2D)           (None, 512, 28, 28)       2359808
    _________________________________________________________________
    max_pooling2d_4 (MaxPooling2 (None, 512, 14, 14)       0
    _________________________________________________________________
    zero_padding2d_11 (ZeroPaddi (None, 512, 16, 16)       0
    _________________________________________________________________
    conv2d_11 (Conv2D)           (None, 512, 14, 14)       2359808
    _________________________________________________________________
    zero_padding2d_12 (ZeroPaddi (None, 512, 16, 16)       0
    _________________________________________________________________
    conv2d_12 (Conv2D)           (None, 512, 14, 14)       2359808
    _________________________________________________________________
    zero_padding2d_13 (ZeroPaddi (None, 512, 16, 16)       0
    _________________________________________________________________
    conv2d_13 (Conv2D)           (None, 512, 14, 14)       2359808
    _________________________________________________________________
    max_pooling2d_5 (MaxPooling2 (None, 512, 7, 7)         0
    _________________________________________________________________
    flatten_1 (Flatten)          (None, 25088)             0
    _________________________________________________________________
    dense_2 (Dense)              (None, 4096)              102764544
    _________________________________________________________________
    dropout_1 (Dropout)          (None, 4096)              0
    _________________________________________________________________
    dense_3 (Dense)              (None, 4096)              16781312
    _________________________________________________________________
    dropout_2 (Dropout)          (None, 4096)              0
    _________________________________________________________________
    dense_4 (Dense)              (None, 1000)              4097000
    =================================================================
    Total params: 138,357,544
    Trainable params: 138,357,544
    Non-trainable params: 0
    _________________________________________________________________

```python
model.pop()
for layer in model.layers:
    layer.trainable=False
```

**Careful!** Now that we've modified the definition of *model*, be careful not to rerun any code in the previous sections, without first recreating the model from scratch! (Yes, I made that mistake myself, which is why I'm warning you about it now...)

Now we're ready to add our new final layer...

```python
model.add(Dense(2, activation='softmax'))
```

```python
??vgg.finetune
```

...and compile our updated model, and set up our batches to use the preprocessed images (note that now we will also *shuffle* the training batches, to add more randomness when using multiple epochs):

```python
gen=image.ImageDataGenerator()
batches = gen.flow(trn_data, trn_labels, batch_size=batch_size, shuffle=True)
val_batches = gen.flow(val_data, val_labels, batch_size=batch_size, shuffle=False)
```

定义一个非常简单的函数去训练模型：

```python
def fit_model(model, batches, val_batches, n_epoch=1):
    model.fit_generator(batches,steps_per_epoch=val_batches.n//batch_size,epochs=n_epoch,
                        validation_data=val_batches, validation_steps=val_batches.n//batch_size)
```

...and now we can use it to train the last layer of our model!

...这个函数我们用来训练模型的最后一层！

(It runs quite slowly, since it still has to calculate all the previous layers in order to know what input to pass to the new final layer. We could precalculate the output of the penultimate layer, like we did for the final layer earlier - but since we're only likely to want one or two iterations, it's easier to follow this alternative approach.)

```python
opt = RMSprop(lr=0.1)
model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])
```

```python
fit_model(model, batches, val_batches, n_epoch=2)
```

    Epoch 1/2
    125/125 [==============================] - 31s 252ms/step - loss: 4.2423 - acc: 0.7260 - val_loss: 2.5987 - val_acc: 0.8330
    Epoch 2/2
    125/125 [==============================] - 30s 243ms/step - loss: 4.0855 - acc: 0.7420 - val_loss: 4.9287 - val_acc: 0.6915

Before moving on, go back and look at how little code we had to write in this section to finetune the model. Because this is such an important and common operation, keras is set up to make it as easy as possible. We didn't even have to use any external helper functions in this section.

It's a good idea to save weights of all your models, so you can re-use them later. Be sure to note the git log number of your model when keeping a research journal of your results.

随时保存之际训练的模型，以便在需要的时候重新使用。

```python
# 保存训练模型
model.save_weights(model_path+'finetune1.h5')
```

```python
# 读取训练模型
model.load_weights(model_path+'finetune1.h5')
```

```python
model.evaluate(val_data, val_labels) # 评估模型
```

    2000/2000 [==============================] - 16s 8ms/step

    [4.928707074165344, 0.6915]

We can look at the earlier prediction examples visualizations by redefining *probs* and *preds* and re-using our earlier code.

```python
preds = model.predict_classes(val_data, batch_size=batch_size)
probs = model.predict_proba(val_data, batch_size=batch_size)[:,0]
probs[:8]
```

    array([0., 1., 1., 0., 0., 0., 0., 0.], dtype=float32)

```python
cm = confusion_matrix(val_classes, preds)
```

```python
plot_confusion_matrix(cm, {'cat':0, 'dog':1})
```

## 训练更多层

Now that we've fine-tuned the new final layer, can we, and should we, fine-tune *all* the dense layers? The answer to both questions, it turns out, is: yes! Let's start with the "can we" question...

现在，我们微调的新的最后一层，我们能够微调所有的全链接层吗？

### 反向传播介绍

The key to training multiple layers of a model, rather than just one, lies in a technique called "back-propagation" (or *backprop* to its friends). Backprop is one of the many words in deep learning parlance that is creating a new word for something that already exists - in this case, backprop simply refers to calculating gradients using the *chain rule*. (But we will still introduce the deep learning terms during this course, since it's important to know them when reading about or discussing deep learning.)

As you (hopefully!) remember from high school, the chain rule is how you calculate the gradient of a "function of a function"--something of the form *f(u), where u=g(x)*. For instance, let's say your function is ```pow((2*x), 2)```. Then u is ```2*x```, and f(u) is ```power(u, 2)```. The chain rule tells us that the derivative of this is simply the product of the derivatives of f() and g(). Using *f'(x)* to refer to the derivative, we can say that: ```f'(x) = f'(u) * g'(x) = 2*u * 2 = 2*(2*x) * 2 = 8*x```.

Let's check our calculation:

```python
# sympy let's us do symbolic differentiation (and much more!) in python
import sympy as sp
# we have to define our variables
x = sp.var('x')
# then we can request the derivative or any expression of that variable
pow(2*x,2).diff()
```

    8*x

The key insight is that the stacking of linear functions and non-linear activations we learnt about in the last section is simply defining a function of functions (of functions, of functions...). Each layer is taking the output of the previous layer's function, and using it as input into its function. Therefore, we can calculate the derivative at any layer by simply multiplying the gradients of that layer and all of its following layers together! This use of the chain rule to allow us to rapidly calculate the derivatives of our model at any layer is referred to as *back propagation*.

The good news is that you'll never have to worry about the details of this yourself, since libraries like Theano and Tensorflow (and therefore wrappers like Keras) provide *automatic differentiation* (or *AD*). ***TODO***

### 在Keras中训练多个层

The code below will work on any model that contains dense layers; it's not just for this VGG model.

NB: Don't skip the step of fine-tuning just the final layer first, since otherwise you'll have one layer with random weights, which will cause the other layers to quickly move a long way from their optimized imagenet weights.

```python
layers = model.layers
```

```python
layers
```

    [<keras.layers.core.Lambda at 0x274f0d41208>,
     <keras.layers.convolutional.ZeroPadding2D at 0x274f0d41470>,
     <keras.layers.convolutional.Conv2D at 0x274f0d41438>,
     <keras.layers.convolutional.ZeroPadding2D at 0x274f0d414e0>,
     <keras.layers.convolutional.Conv2D at 0x274f0d4e3c8>,
     <keras.layers.pooling.MaxPooling2D at 0x274f0d4e978>,
     <keras.layers.convolutional.ZeroPadding2D at 0x274f0d7d5f8>,
     <keras.layers.convolutional.Conv2D at 0x274f0d7de10>,
     <keras.layers.convolutional.ZeroPadding2D at 0x274f0f22b70>,
     <keras.layers.convolutional.Conv2D at 0x274f0f375c0>,
     <keras.layers.pooling.MaxPooling2D at 0x274f0f378d0>,
     <keras.layers.convolutional.ZeroPadding2D at 0x274f0f614e0>,
     <keras.layers.convolutional.Conv2D at 0x274f0f61cf8>,
     <keras.layers.convolutional.ZeroPadding2D at 0x274f0f75a58>,
     <keras.layers.convolutional.Conv2D at 0x274f0f89400>,
     <keras.layers.convolutional.ZeroPadding2D at 0x274f0f89710>,
     <keras.layers.convolutional.Conv2D at 0x274ffaf1240>,
     <keras.layers.pooling.MaxPooling2D at 0x274ffaf14e0>,
     <keras.layers.convolutional.ZeroPadding2D at 0x274ffb1d080>,
     <keras.layers.convolutional.Conv2D at 0x274ffb1dac8>,
     <keras.layers.convolutional.ZeroPadding2D at 0x274ffb2fdd8>,
     <keras.layers.convolutional.Conv2D at 0x274ffb48048>,
     <keras.layers.convolutional.ZeroPadding2D at 0x274ffb482e8>,
     <keras.layers.convolutional.Conv2D at 0x274ffb56d68>,
     <keras.layers.pooling.MaxPooling2D at 0x274ffb6f0b8>,
     <keras.layers.convolutional.ZeroPadding2D at 0x274ffb834e0>,
     <keras.layers.convolutional.Conv2D at 0x274ffb9de80>,
     <keras.layers.convolutional.ZeroPadding2D at 0x274ffbafcc0>,
     <keras.layers.convolutional.Conv2D at 0x274ffbafc18>,
     <keras.layers.convolutional.ZeroPadding2D at 0x274ffbc4ef0>,
     <keras.layers.convolutional.Conv2D at 0x27582ae9978>,
     <keras.layers.pooling.MaxPooling2D at 0x27582ae9e80>,
     <keras.layers.core.Flatten at 0x27582b18080>,
     <keras.layers.core.Dense at 0x27582b18d30>,
     <keras.layers.core.Dropout at 0x27582b3fda0>,
     <keras.layers.core.Dense at 0x27582b3f908>,
     <keras.layers.core.Dropout at 0x27582b8ce48>,
     <keras.layers.core.Dense at 0x275a360e048>]

```python
[index for index,layer in enumerate(layers) if type(layer) is Dense][0]
```

    33

```python
# Get the index of the first dense layer...
first_dense_idx = [index for index,layer in enumerate(layers) if type(layer) is Dense][0]
# ...and set this and all subsequent layers to trainable
for layer in layers[first_dense_idx:]:
    layer.trainable=True
```

Since we haven't changed our architecture, there's no need to re-compile the model - instead, we just set the learning rate. Since we're training more layers, and since we've already optimized the last layer, we should use a lower learning rate than previously.

我们这样就改变了模型结构，这里并不需要重新编译模型，而是仅仅更改学习率。因为我们需要训练更多的层，同时我们已经训练好了最后一层，所以我们需要一个比之前更低的学习率。

```python
K.set_value(opt.lr, 0.01)
fit_model(model, batches, val_batches, 3)
```

    Epoch 1/3
      1/125 [..............................] - ETA: 19s - loss: 6.0443 - acc: 0.6250

    C:\Anaconda3\lib\site-packages\keras\engine\training.py:975: UserWarning: Discrepancy between trainable weights and collected trainable weights, did you set `model.trainable` without calling `model.compile` after ?
      'Discrepancy between trainable weights and collected trainable'

    125/125 [==============================] - 31s 245ms/step - loss: 4.2942 - acc: 0.7305 - val_loss: 3.4219 - val_acc: 0.7835
    Epoch 2/3
    125/125 [==============================] - 31s 245ms/step - loss: 3.1749 - acc: 0.7990 - val_loss: 2.3067 - val_acc: 0.8550
    Epoch 3/3
    125/125 [==============================] - 31s 245ms/step - loss: 2.4665 - acc: 0.8450 - val_loss: 2.2643 - val_acc: 0.8545

This is an extraordinarily powerful 5 lines of code. We have fine-tuned all of our dense layers to be optimized for our specific data set. This kind of technique has only become accessible in the last year or two - and we can already do it in just 5 lines of python!

```python
model.save_weights(model_path+'finetune2.h5')
```

There's generally little room for improvement in training the convolutional layers, if you're using the model on natural images (as we are). However, there's no harm trying a few of the later conv layers, since it may give a slight improvement, and can't hurt (and we can always load the previous weights if the accuracy decreases).

```python
for layer in layers[12:]:
    layer.trainable=True
K.set_value(opt.lr, 0.001)
```

```python
fit_model(model, batches, val_batches, 4)
```

    Epoch 1/4
      1/125 [..............................] - ETA: 17s - loss: 2.0148 - acc: 0.8750

    C:\Anaconda3\lib\site-packages\keras\engine\training.py:975: UserWarning: Discrepancy between trainable weights and collected trainable weights, did you set `model.trainable` without calling `model.compile` after ?
      'Discrepancy between trainable weights and collected trainable'

    125/125 [==============================] - 31s 246ms/step - loss: 2.4793 - acc: 0.8430 - val_loss: 2.2642 - val_acc: 0.8555
    Epoch 2/4
    125/125 [==============================] - 31s 247ms/step - loss: 2.5657 - acc: 0.8380 - val_loss: 2.2650 - val_acc: 0.8550
    Epoch 3/4
    125/125 [==============================] - 31s 247ms/step - loss: 2.1890 - acc: 0.8605 - val_loss: 2.2708 - val_acc: 0.8560
    Epoch 4/4
    125/125 [==============================] - 31s 249ms/step - loss: 2.3187 - acc: 0.8510 - val_loss: 2.2617 - val_acc: 0.8570

```python
model.save_weights(model_path+'finetune3.h5')
```

You can always load the weights later and use the model to do whatever you need:

```python
model.load_weights(model_path+'finetune2.h5')
model.evaluate_generator(get_batches(path+'valid', gen, False, batch_size*2), val_batches.n)
```

    Found 2000 images belonging to 2 classes.

    [2.2618036316395655, 0.8546390778533636]

```python
val_batches.n
```

    2000

```python

```


# Quaternion Series Quantum Mechanics: Lectures 1 and 2

by Doug Sweetser, email to sweetser@alum.mit.edu

This notebook is being created as a companion to the book "Quantum Mechanics: the Theoretical Minimum" by Susskind and Friedman (QM:TTM for short). Those authors of course never use quaternions as they are a bit player in the crowded field of mathematical tools. Nature has used one accounting system since the beginning of space-time, so I will be a jerk in the name of consistency. This leads to a different perspective on what makes an equation quantum mechanical. If a conjugate operator is used, then the expression is about quantum mechanics. It is odd to have such a brief assertion given the complexity of the subject, but that make the hypothesis fun - and testable by seeing if anything in the book cannot be done with quaternions and their conjugates. Import the tools to work with quaternions in this notebook.

```python
%%capture
%matplotlib inline
import numpy as np
import sympy as sp
import matplotlib.pyplot as plt

# To get equations the look like, well, equations, use the following.
from sympy.interactive import printing
printing.init_printing(use_latex=True)
from IPython.display import display

# Tools for manipulating quaternions.
import Q_tools as qt;
```

## Lecture 1: Systems and Experiments

### Bracket Notation and Three Identities

Bracket notation from this quaternion-centric perspective is just a quaternion product where the first term must necessarily be conjugated. I have called this the "Euclidean product". The quaternion product is associative but the Euclidean product is not ($(A^* B)^* C \ne A^* (B^* C)$ although their norms are equal). Write out three things in bracket notation that are known to be true about inner products(QM:TTH, p. 31).
1. $<A|A> \rightarrow A^* A$ is real
1. $<A|B> = <B|A>^* \rightarrow A^* B = (B^* A)^*$
1. $(<A|+<B|)|C> = <A|C> + <B|C> \rightarrow (A+ B)^*C = A^*C + B^* C$

This may provide the first signs that the odd math of quantum mechanics is the math of Euclidean products of quaternions.

So, is $A^* A$ real? Yes and no.

```python
a0, A1, A2, A3 = sp.symbols("a0 A1 A2 A3")
b0, B1, B2, B3 = sp.symbols("b0 B1 B2 B3")
c0, C1, C2, C3 = sp.symbols("c0 C1 C2 C3")
A = qt.QH([a0, A1, A2, A3], qtype="A")
B = qt.QH([b0, B1, B2, B3], qtype="B")
C = qt.QH([c0, C1, C2, C3], qtype="C")
display(A.conj().product(A).t)
display(A.conj().product(A).x)
display(A.conj().product(A).y)
display(A.conj().product(A).z)
```

The first term is a real-valued, with the 3-imaginary vector equal to zero. I think it is bad practice to just pretend the three zeros are *not there in any way*. One can make an equivalence relation between quaternions of the form $(\mathbb{R}, 0, 0, 0)$ and the real numbers. The real numbers are a subgroup of quaternions, and never the other way around.

It is important to understand exactly why the three imaginary terms are zero. It is too common for people to say "it's the norm" and give the subject no thought. No thought means no insights. A quaternion points in the direction of itself, so all the anti-symmetric cross terms are equal to zero. The conjugate operator picks out the mirror reflection of the imaginary terms. The product of an imaginary with its mirror image is an all positive real number and zero for all three imaginary numbers.

Calculus is the story of neighborhoods near points. There are two broad classes of changes one can imagine for a norm. In the first, a point $A$ goes to $A'$. It could be either slightly bigger or smaller, shown in a slightly bigger or smaller first value. Or the mirror reflection to be slightly off. This would create a non-zero space-times-time 3-vector. Everyone accepts that a norm can get larger or smaller, it is a "size" thing. But a change in direction will lead to imaginary terms that can either commute, anti-commute, or be a mixture of both. This possibility makes this view of a quaternion norm sound richer.

Test out the second identity:

$$(A^* B)^* = (B^*, A)$$

```python
AB_conj = A.Euclidean_product(B)
BA = B.Euclidean_product(A).conj()
print("(A* B)* = {}".format(AB_conj))
print("B* A    = {}".format(BA))
print("(A* B)* - B* A = {}".format(AB_conj.dif(BA)))
```

    (A* B)* = (A1*B1 + A2*B2 + A3*B3 + a0*b0, -A1*b0 - A2*B3 + A3*B2 + B1*a0, A1*B3 - A2*b0 - A3*B1 + B2*a0, -A1*B2 + A2*B1 - A3*b0 + B3*a0) A*xB
    B* A    = (A1*B1 + A2*B2 + A3*B3 + a0*b0, -A1*b0 - A2*B3 + A3*B2 + B1*a0, A1*B3 - A2*b0 - A3*B1 + B2*a0, -A1*B2 + A2*B1 - A3*b0 + B3*a0) B*xA*
    (A* B)* - B* A = (0, 0, 0, 0) A*xB-B*xA*

Note on notation: someone pointed out that is *absolutely all calculations start and end with quaternions*, then it is easy to feel lost - this quaternion looks like that one. The string at the end that I call a "qtype" represents all the steps that went into a calculation. The last qtype above reads: A*xB-B*xA* which hopefully is clear in this contex.

Despite the fact that quaternions do not commute, the conjugate operator does the job correctly because the angle between the two quaternions does not change.

Now for the third identity about sums.

```python
A_plus_B_then_C = A.conj().add(B.conj()).product(C).expand_q()
AC_plus_BC = A.conj().product(C).add(B.conj().product(C)).expand_q()
print("(A+B)* C:  {}\n".format(A_plus_B_then_C))
print("A*C + B*C: {}\n".format(AC_plus_BC))
print("(A+B)* C - (A*C + B*C): {}".format(A_plus_B_then_C.dif(AC_plus_BC)))
```

    (A+B)* C:  (A1*C1 + A2*C2 + A3*C3 + B1*C1 + B2*C2 + B3*C3 + a0*c0 + b0*c0, -A1*c0 - A2*C3 + A3*C2 - B1*c0 - B2*C3 + B3*C2 + C1*a0 + C1*b0, A1*C3 - A2*c0 - A3*C1 + B1*C3 - B2*c0 - B3*C1 + C2*a0 + C2*b0, -A1*C2 + A2*C1 - A3*c0 - B1*C2 + B2*C1 - B3*c0 + C3*a0 + C3*b0) A*+B*xC

    A*C + B*C: (A1*C1 + A2*C2 + A3*C3 + B1*C1 + B2*C2 + B3*C3 + a0*c0 + b0*c0, -A1*c0 - A2*C3 + A3*C2 - B1*c0 - B2*C3 + B3*C2 + C1*a0 + C1*b0, A1*C3 - A2*c0 - A3*C1 + B1*C3 - B2*c0 - B3*C1 + C2*a0 + C2*b0, -A1*C2 + A2*C1 - A3*c0 - B1*C2 + B2*C1 - B3*c0 + C3*a0 + C3*b0) A*xC+B*xC

    (A+B)* C - (A*C + B*C): (0, 0, 0, 0) A*+B*xC-A*xC+B*xC

Subtracting one from the other shows they are identical.

There are many more algebraic relationships known for Hilbert spaces such as the triangle inequality and the Schwarz inequality which is the basis of the uncertainty principle. These all work for the [Euclidean product with quaternions](https://dougsweetser.github.io/Q/QM/bracket_notation/).

## Lecture 2: Quantum States

### Quaternion Series as Quantum States

A quantum state is an n-dimensional vector space. This is fundamentally different from a set of states because certain math relationships are allowed. Vectors can be added to one another, multiplied by complex numbers. One can take the inner product of two vectors. Most important calculations involve taking the inner product.

A perspective I will explore here is that a (possibly infinite) series of quaternions has the same algebraic properties of Hilbert spaces when one uses the Euclidean product, $A^* B = \sum_{1}^{n} a_n^* b_n$

This only works if the length of the series for **A** is exactly equal to that of **B**. Whatever can be done with a quaternion can be done with its series representation. Unlike vectors that can either be be a row or a column, quaternion series only have a length. Let's just do one calculation, < A | A >:

```python
A = qt.QHStates([qt.QH([0,1,2,3]), qt.QH([1,2,1,2])])
AA = A.Euclidean_product('bra', ket=A)
AA.print_states("<A|A>")
```

    <A|A>
    n=1: (14, 0, 0, 0) Q*xQ
    n=2: (10, 0, 0, 0) Q*xQ
    sum= (24, 0, 0, 0) Q*xQ+Q*xQ

A little calculation in the head should show this works as expected - except one is not used to seeing quaternion series in action.

The first system analyzed has but 2 states, keeping things simple. The first pair of states are likewise so simple they are orthonormal to a casual observer.

```python
q0, q1, qi, qj, qk = qt.QH().q_0(), qt.QH().q_1(), qt.QH().q_i(), qt.QH().q_j(), qt.QH().q_k()

u = qt.QHStates([q1, q0])
d = qt.QHStates([q0, q1])

u.print_states("u", True)
d.print_states("d")
```

    u
    n=1: (1, 0, 0, 0) 1
    n=2: (0, 0, 0, 0) 0
    sum= (1, 0, 0, 0) 1+0

    d
    n=1: (0, 0, 0, 0) 0
    n=2: (1, 0, 0, 0) 1
    sum= (1, 0, 0, 0) 0+1

Calculate $<u|u>$, $<d|d>$ and $<u|d>$:

```python
u.Euclidean_product('bra', ket=u).print_states("<u|u>")
```

    <u|u>
    n=1: (1, 0, 0, 0) 1*x1
    n=2: (0, 0, 0, 0) 0*x0
    sum= (1, 0, 0, 0) 1*x1+0*x0

```python
d.Euclidean_product('bra', ket=d).print_states("<d|d>")
```

    <d|d>
    n=1: (0, 0, 0, 0) 0*x0
    n=2: (1, 0, 0, 0) 1*x1
    sum= (1, 0, 0, 0) 0*x0+1*x1

```python
u.Euclidean_product('bra', ket=d).print_states("<u|d>")
```

    <u|d>
    n=1: (0, 0, 0, 0) 1*x0
    n=2: (0, 0, 0, 0) 0*x1
    sum= (0, 0, 0, 0) 1*x0+0*x1

The next pair of states is constructed from the first pair, $u$ and $d$ like so (QM:TTM, page 41):

```python
sqrt_2op = qt.QHStates([qt.QH([sp.sqrt(1/2), 0, 0, 0])])

u2 = u.Euclidean_product('ket', operator=sqrt_2op)
d2 = d.Euclidean_product('ket', operator=sqrt_2op)

r = u2.add(d2)
L = u2.dif(d2)

r.print_states("r", True)
L.print_states("L")
```

    r
    n=1: (0.707106781186548, 0, 0, 0) Qx1+0x0+Qx0+0x1
    n=2: (0.707106781186548, 0, 0, 0) 0x1+Qx0+0x0+Qx1
    sum= (1.41421356237310, 0, 0, 0) Qx1+0x0+Qx0+0x1+0x1+Qx0+0x0+Qx1

    L
    n=1: (0.707106781186548, 0, 0, 0) Qx1+0x0-Qx0+0x1
    n=2: (-0.707106781186548, 0, 0, 0) 0x1+Qx0-0x0+Qx1
    sum= (0, 0, 0, 0) Qx1+0x0-Qx0+0x1+0x1+Qx0-0x0+Qx1

```python
r.Euclidean_product('bra', ket=r).print_states("<r|r>", True)
L.Euclidean_product('bra', ket=L).print_states("<L|L>", True)
r.Euclidean_product('bra', ket=L).print_states("<r|L>", True)
```

    <r|r>
    n=1: (0.500000000000000, 0, 0, 0) Qx1+0x0+Qx0+0x1*xQx1+0x0+Qx0+0x1
    n=2: (0.500000000000000, 0, 0, 0) 0x1+Qx0+0x0+Qx1*x0x1+Qx0+0x0+Qx1
    sum= (1.00000000000000, 0, 0, 0) Qx1+0x0+Qx0+0x1*xQx1+0x0+Qx0+0x1+0x1+Qx0+0x0+Qx1*x0x1+Qx0+0x0+Qx1

    <L|L>
    n=1: (0.500000000000000, 0, 0, 0) Qx1+0x0-Qx0+0x1*xQx1+0x0-Qx0+0x1
    n=2: (0.500000000000000, 0, 0, 0) 0x1+Qx0-0x0+Qx1*x0x1+Qx0-0x0+Qx1
    sum= (1.00000000000000, 0, 0, 0) Qx1+0x0-Qx0+0x1*xQx1+0x0-Qx0+0x1+0x1+Qx0-0x0+Qx1*x0x1+Qx0-0x0+Qx1

    <r|L>
    n=1: (0.500000000000000, 0, 0, 0) Qx1+0x0+Qx0+0x1*xQx1+0x0-Qx0+0x1
    n=2: (-0.500000000000000, 0, 0, 0) 0x1+Qx0+0x0+Qx1*x0x1+Qx0-0x0+Qx1
    sum= (0, 0, 0, 0) Qx1+0x0+Qx0+0x1*xQx1+0x0-Qx0+0x1+0x1+Qx0+0x0+Qx1*x0x1+Qx0-0x0+Qx1

The final calculation for chapter 2 is like the one for $r$ and $L$ except one uses an arbitrarily chosen imaginary value - it could point any direction in 3D space - like so:

```python
i_op = qt.QHStates([q1, q0, q0, qi])

i = r.Euclidean_product('ket', operator=i_op)
o = L.Euclidean_product('ket', operator=i_op)

i.print_states("i", True)
o.print_states("o")
```

    i
    n=1: (0.707106781186548, 0, 0, 0) 1xQx1+0x0+Qx0+0x1+0x0x1+Qx0+0x0+Qx1
    n=2: (0, 0.707106781186548, 0, 0) 0xQx1+0x0+Qx0+0x1+ix0x1+Qx0+0x0+Qx1
    sum= (0.707106781186548, 0.707106781186548, 0, 0) 1xQx1+0x0+Qx0+0x1+0x0x1+Qx0+0x0+Qx1+0xQx1+0x0+Qx0+0x1+ix0x1+Qx0+0x0+Qx1

    o
    n=1: (0.707106781186548, 0, 0, 0) 1xQx1+0x0-Qx0+0x1+0x0x1+Qx0-0x0+Qx1
    n=2: (0, -0.707106781186548, 0, 0) 0xQx1+0x0-Qx0+0x1+ix0x1+Qx0-0x0+Qx1
    sum= (0.707106781186548, -0.707106781186548, 0, 0) 1xQx1+0x0-Qx0+0x1+0x0x1+Qx0-0x0+Qx1+0xQx1+0x0-Qx0+0x1+ix0x1+Qx0-0x0+Qx1

```python
i.Euclidean_product('bra', ket=i).print_states("<i|i>", True)
o.Euclidean_product('bra', ket=o).print_states("<o|o>", True)
i.Euclidean_product('bra', ket=o).print_states("<i|o>")
```

    <i|i>
    n=1: (0.500000000000000, 0, 0, 0) 1xQx1+0x0+Qx0+0x1+0x0x1+Qx0+0x0+Qx1*x1xQx1+0x0+Qx0+0x1+0x0x1+Qx0+0x0+Qx1
    n=2: (0.500000000000000, 0, 0, 0) 0xQx1+0x0+Qx0+0x1+ix0x1+Qx0+0x0+Qx1*x0xQx1+0x0+Qx0+0x1+ix0x1+Qx0+0x0+Qx1
    sum= (1.00000000000000, 0, 0, 0) 1xQx1+0x0+Qx0+0x1+0x0x1+Qx0+0x0+Qx1*x1xQx1+0x0+Qx0+0x1+0x0x1+Qx0+0x0+Qx1+0xQx1+0x0+Qx0+0x1+ix0x1+Qx0+0x0+Qx1*x0xQx1+0x0+Qx0+0x1+ix0x1+Qx0+0x0+Qx1

    <o|o>
    n=1: (0.500000000000000, 0, 0, 0) 1xQx1+0x0-Qx0+0x1+0x0x1+Qx0-0x0+Qx1*x1xQx1+0x0-Qx0+0x1+0x0x1+Qx0-0x0+Qx1
    n=2: (0.500000000000000, 0, 0, 0) 0xQx1+0x0-Qx0+0x1+ix0x1+Qx0-0x0+Qx1*x0xQx1+0x0-Qx0+0x1+ix0x1+Qx0-0x0+Qx1
    sum= (1.00000000000000, 0, 0, 0) 1xQx1+0x0-Qx0+0x1+0x0x1+Qx0-0x0+Qx1*x1xQx1+0x0-Qx0+0x1+0x0x1+Qx0-0x0+Qx1+0xQx1+0x0-Qx0+0x1+ix0x1+Qx0-0x0+Qx1*x0xQx1+0x0-Qx0+0x1+ix0x1+Qx0-0x0+Qx1

    <i|o>
    n=1: (0.500000000000000, 0, 0, 0) 1xQx1+0x0+Qx0+0x1+0x0x1+Qx0+0x0+Qx1*x1xQx1+0x0-Qx0+0x1+0x0x1+Qx0-0x0+Qx1
    n=2: (-0.500000000000000, 0, 0, 0) 0xQx1+0x0+Qx0+0x1+ix0x1+Qx0+0x0+Qx1*x0xQx1+0x0-Qx0+0x1+ix0x1+Qx0-0x0+Qx1
    sum= (0, 0, 0, 0) 1xQx1+0x0+Qx0+0x1+0x0x1+Qx0+0x0+Qx1*x1xQx1+0x0-Qx0+0x1+0x0x1+Qx0-0x0+Qx1+0xQx1+0x0+Qx0+0x1+ix0x1+Qx0+0x0+Qx1*x0xQx1+0x0-Qx0+0x1+ix0x1+Qx0-0x0+Qx1

Notice how long the qtypes have gotten (the strings that keep a record of all the manipulations done to a quaternion). The initial state was just a zero and a one, but that had to get added to another and normalized, then multiplied by a factor of $i$ and combined again.

Orthonormal again, as hoped for.

Is the quaternion series approach a faithful representation of these 6 states? On page 43-44, there are 8 products that all add up to one half. See if this works out...

```python
ou = o.Euclidean_product('bra', ket=u)
uo = i.Euclidean_product('bra', ket=o)
print("ouuo sum:\n", ou.product('bra', ket=uo).summation(), "\n")
od = o.Euclidean_product('bra', ket=d)
do = d.Euclidean_product('bra', ket=o)
print("oddo sum:\n", od.product('bra', ket=do).summation(), "\n")
iu = i.Euclidean_product('bra', ket=u)
ui = u.Euclidean_product('bra', ket=i)
print("iuui sum:\n", iu.product('bra', ket=ui).summation(), "\n")
id = i.Euclidean_product('bra', ket=d)
di = d.Euclidean_product('bra', ket=i)
print("iddi sum:\n", id.product('bra', ket=di).summation())
```

    ouuo sum:
     (0.353553390593274, 0, 0, 0) 1xQx1+0x0-Qx0+0x1+0x0x1+Qx0-0x0+Qx1*x1x1xQx1+0x0+Qx0+0x1+0x0x1+Qx0+0x0+Qx1*x1xQx1+0x0-Qx0+0x1+0x0x1+Qx0-0x0+Qx1+0xQx1+0x0-Qx0+0x1+ix0x1+Qx0-0x0+Qx1*x0x0xQx1+0x0+Qx0+0x1+ix0x1+Qx0+0x0+Qx1*x0xQx1+0x0-Qx0+0x1+ix0x1+Qx0-0x0+Qx1

    oddo sum:
     (0.500000000000000, 0, 0, 0) 1xQx1+0x0-Qx0+0x1+0x0x1+Qx0-0x0+Qx1*x0x0*x1xQx1+0x0-Qx0+0x1+0x0x1+Qx0-0x0+Qx1+0xQx1+0x0-Qx0+0x1+ix0x1+Qx0-0x0+Qx1*x1x1*x0xQx1+0x0-Qx0+0x1+ix0x1+Qx0-0x0+Qx1

    iuui sum:
     (0.500000000000000, 0, 0, 0) 1xQx1+0x0+Qx0+0x1+0x0x1+Qx0+0x0+Qx1*x1x1*x1xQx1+0x0+Qx0+0x1+0x0x1+Qx0+0x0+Qx1+0xQx1+0x0+Qx0+0x1+ix0x1+Qx0+0x0+Qx1*x0x0*x0xQx1+0x0+Qx0+0x1+ix0x1+Qx0+0x0+Qx1

    iddi sum:
     (0.500000000000000, 0, 0, 0) 1xQx1+0x0+Qx0+0x1+0x0x1+Qx0+0x0+Qx1*x0x0*x1xQx1+0x0+Qx0+0x1+0x0x1+Qx0+0x0+Qx1+0xQx1+0x0+Qx0+0x1+ix0x1+Qx0+0x0+Qx1*x1x1*x0xQx1+0x0+Qx0+0x1+ix0x1+Qx0+0x0+Qx1

```python
Or = o.Euclidean_product('bra', ket=r)
ro = r.Euclidean_product('bra', ket=o)
print("orro:\n", Or.product('bra', ket=ro).summation(), "\n")
oL = o.Euclidean_product('bra', ket=L)
Lo = L.Euclidean_product('bra', ket=o)
print("oLLo:\n", oL.product('bra', ket=Lo).summation(), "\n")
ir = i.Euclidean_product('bra', ket=r)
ri = r.Euclidean_product('bra', ket=i)
print("irri:\n", ir.product('bra', ket=ri).summation(), "\n")
iL = i.Euclidean_product('bra', ket=L)
Li = L.Euclidean_product('bra', ket=i)
print("iLLi:\n", iL.product('bra', ket=Li).summation())
```

    orro:
     (0.500000000000000, 0, 0, 0) 1xQx1+0x0-Qx0+0x1+0x0x1+Qx0-0x0+Qx1*xQx1+0x0+Qx0+0x1xQx1+0x0+Qx0+0x1*x1xQx1+0x0-Qx0+0x1+0x0x1+Qx0-0x0+Qx1+0xQx1+0x0-Qx0+0x1+ix0x1+Qx0-0x0+Qx1*x0x1+Qx0+0x0+Qx1x0x1+Qx0+0x0+Qx1*x0xQx1+0x0-Qx0+0x1+ix0x1+Qx0-0x0+Qx1

    oLLo:
     (0.500000000000000, 0, 0, 0) 1xQx1+0x0-Qx0+0x1+0x0x1+Qx0-0x0+Qx1*xQx1+0x0-Qx0+0x1xQx1+0x0-Qx0+0x1*x1xQx1+0x0-Qx0+0x1+0x0x1+Qx0-0x0+Qx1+0xQx1+0x0-Qx0+0x1+ix0x1+Qx0-0x0+Qx1*x0x1+Qx0-0x0+Qx1x0x1+Qx0-0x0+Qx1*x0xQx1+0x0-Qx0+0x1+ix0x1+Qx0-0x0+Qx1

    irri:
     (0.500000000000000, 0, 0, 0) 1xQx1+0x0+Qx0+0x1+0x0x1+Qx0+0x0+Qx1*xQx1+0x0+Qx0+0x1xQx1+0x0+Qx0+0x1*x1xQx1+0x0+Qx0+0x1+0x0x1+Qx0+0x0+Qx1+0xQx1+0x0+Qx0+0x1+ix0x1+Qx0+0x0+Qx1*x0x1+Qx0+0x0+Qx1x0x1+Qx0+0x0+Qx1*x0xQx1+0x0+Qx0+0x1+ix0x1+Qx0+0x0+Qx1

    iLLi:
     (0.500000000000000, 0, 0, 0) 1xQx1+0x0+Qx0+0x1+0x0x1+Qx0+0x0+Qx1*xQx1+0x0-Qx0+0x1xQx1+0x0-Qx0+0x1*x1xQx1+0x0+Qx0+0x1+0x0x1+Qx0+0x0+Qx1+0xQx1+0x0+Qx0+0x1+ix0x1+Qx0+0x0+Qx1*x0x1+Qx0-0x0+Qx1x0x1+Qx0-0x0+Qx1*x0xQx1+0x0+Qx0+0x1+ix0x1+Qx0+0x0+Qx1

There is an important technical detail in this calculation I should point out. In the <bra|ket> form, the bra gets conjugated. Notice though that if one does two of these, < i | L >< L | i >, then there has to be a product formed between the two brackets. In practice, < i | L >* < L | i > gives the wrong result:

```python
print("iL*Li:\n", iL.Euclidean_product('bra', ket=Li).summation())
```

    iL*Li:
     (0, 0, 0, 0) 1xQx1+0x0+Qx0+0x1+0x0x1+Qx0+0x0+Qx1*xQx1+0x0-Qx0+0x1*xQx1+0x0-Qx0+0x1*x1xQx1+0x0+Qx0+0x1+0x0x1+Qx0+0x0+Qx1+0xQx1+0x0+Qx0+0x1+ix0x1+Qx0+0x0+Qx1*x0x1+Qx0-0x0+Qx1*x0x1+Qx0-0x0+Qx1*x0xQx1+0x0+Qx0+0x1+ix0x1+Qx0+0x0+Qx1

```python

```


This is very much work-in-progress!

[back to overview page](index.ipynb)

```python
import sympy as sp
sp.init_printing()
```

```python
t, w0, w1 = sp.symbols("t omega:2")
```

```python
sine0 = sp.sin(w0 * t)
sine0
```

```python
sine1 = sp.sin(w1 * t)
sine1
```

```python
two_sines = sine0 + sine1
two_sines
```

```python
%matplotlib inline
```

If you don't know what the above line means, go to [Getting Started with matplotlib](../plotting/matplotlib.ipynb).

```python
def plot_two_sines(f0, f1):
    """Plot our two sines with the given frequencies."""
    sp.plot(two_sines.subs({w0: 2 * sp.pi * f0, w1: 2 * sp.pi * f1}), (t, 0, 1))
```

Clearly, this is a superposition of two signals:

```python
plot_two_sines(20, 2)
```

This is something strange:

```python
plot_two_sines(20, 9)
```

And this is clearly an amplitude modulation:

```python
plot_two_sines(60, 64)
```

```python
# TODO: interactive widgets: http://jakevdp.github.io/blog/2013/12/05/static-interactive-widgets/
```

```python
from sympy.simplify.fu import TR9
```

```python
TR9(two_sines)
```

Just to be sure:

```python
_.simplify()
```

```python
# TODO: listen to beating
```

```python
# TODO: save to sound file
```

```python

```

  To the extent possible under law,
  the person who associated CC0
  with this work has waived all copyright and related or neighboring
  rights to this work.