```python
import networkx as nx
import matplotlib.pyplot as plt
import numpy as np
import random
import sympy
from sklearn import preprocessing
```

    Bad key "text.kerning_factor" on line 4 in
    /home/sc/anaconda3/envs/old_nx/lib/python3.6/site-packages/matplotlib/mpl-data/stylelib/_classic_test_patch.mplstyle.
    You probably need to get an updated matplotlibrc file from
    http://github.com/matplotlib/matplotlib/blob/master/matplotlibrc.template
    or from the matplotlib source distribution

```python
max_B = 10000 # maximum budget
no_runs = 100 # no. of runs to average
K = 4 # controls number of restarts

np.random.seed(0)
random.seed(42)
```

```python
with open('citeseer/citeseer.content') as f:
    lines = f.readlines()

ne = preprocessing.LabelEncoder()
ce = preprocessing.LabelEncoder()
nodes_list = [line.strip().split() for line in lines]
```

```python
nodes = [n[0] for n in nodes_list]
classes = [n[-1] for n in nodes_list]
nodes = ne.fit_transform(nodes+['XYZA'])
classes = ce.fit_transform(classes+['XYZA'])
```

```python
G = nx.Graph()
for i,j in enumerate(nodes):
    G.add_node(nodes[i],classes = classes[i])
```

```python
with open('citeseer/citeseer.cites') as f:
    lines = f.readlines()

def replace(star):
    if star in ne.classes_:
        return star
    return 'XYZA'

edges_list = [line.strip().split() for line in lines]
edges_0 = ne.transform([replace(x[0]) for x in edges_list])
edges_1 = ne.transform([replace(x[1]) for x in edges_list])
edges = [(a,b) for a,b in zip(edges_0,edges_1)]
```

```python
G.add_edges_from(edges)
```

```python
print(G.number_of_edges(),G.number_of_nodes())
G = max(nx.connected_component_subgraphs(G), key=len)
```

    4675 3313

```python
from louvain import detect_communities, modularity
def draw_communities(G, node_size=90, alpha=1, k=None, randomized=False):
    partition = detect_communities(G, randomized=randomized)
    print("Modularity for best partition:", modularity(G, partition))
    community_map = {}
    for community, nodes in enumerate(partition):
        for node in nodes:
            community_map[node] = community

    cmap = plt.get_cmap("jet")
    plt.figure(figsize=(15, 15))
    pos = nx.spring_layout(G, k=k)
    indexed = [community_map.get(node) for node in G]
    plt.axis("off")
    nx.draw_networkx_nodes(G, pos=pos, cmap=cmap, node_color=indexed, node_size=node_size, alpha=alpha)
    nx.draw_networkx_edges(G, pos=pos, alpha=0.2)
    #for x in range(len(pos)):
    #    print(pos)
    #    pos[x] = pos[x]+np.array([0.02,0])
    #labels = dict([(n, n) for n in G.nodes()]) #add labels
    #_ = nx.draw_networkx_labels(G, pos, labels=labels, font_color='#000000', font_size=15) #draw labels

draw_communities(G)
```

```python
partition = detect_communities(G, randomized=False)
community_map = {}
node_map = {}

for community, nodes in enumerate(partition):
    for node in nodes:
        node_map[node] = community
    community_map[community] = nodes

G_no_edges=G.number_of_edges()
G_no_nodes=G.number_of_nodes()

print(nx.is_connected(G),G_no_edges,G_no_nodes)
```

    True 3751 2129

```python
def MH_sampling(G,B):
    est_MH= []
    est_MH_t = 0
    sample = np.random.choice(G.nodes())
    est_MH_t += node_fn(sample)
    est_MH.append(est_MH_t)
    for ii in range(2,B+1):
        neighbors = list(nx.neighbors(G,sample))
        sample_t = np.random.choice(neighbors)
        if np.random.rand() <= (G.degree(sample)/G.degree(sample_t)):
            sample = sample_t
        est_MH_t += node_fn(sample)
        est_MH.append(est_MH_t/ii)
    return np.array(est_MH)

def RDS_sampling(G,B):
    est_RW = []
    est_RW_t1 = 0
    est_RW_t2 = 0
    sample = np.random.choice(G.nodes())
    deg_pr_sent = G.degree(sample)
    est_RW_t1 += node_fn(sample)/deg_pr_sent
    est_RW_t2 += 1/deg_pr_sent
    est_RW.append(est_RW_t1/est_RW_t2)
    for ii in range(2,B+1):
        neighbors = list(nx.neighbors(G,sample))
        sample = np.random.choice(neighbors)
        deg_pr_sent = G.degree(sample)
        est_RW_t1 += node_fn(sample)/deg_pr_sent
        est_RW_t2 += 1/deg_pr_sent
        est_RW.append(est_RW_t1/est_RW_t2)
    return np.array(est_RW)
```

```python
def RDSRR_sampling(G,B,U=None):
    #M = np.power(B,2/3)
    #restart_ind = np.power(np.arange(1,M),1.5)
    restart_ind = [10]
    t=20
    while restart_ind[-1]<B:
        restart_ind.append(restart_ind[-1]+int(80*np.log(t)))
        t+=1

    est_RW = []
    est_RW_t1 = 0
    est_RW_t2 = 0
    sample = np.random.choice(G.nodes())
    deg_pr_sent = G.degree(sample)
    est_RW_t1 += node_fn(sample)/deg_pr_sent
    est_RW_t2 += 1/deg_pr_sent
    est_RW.append(est_RW_t1/est_RW_t2)
    for ii in range(2,B+1):
        if ii in restart_ind:
            #sample = np.random.choice(community_map[np.random.randint(len(community_map))])
            if U is None:
                sample = np.random.choice(community_map[np.random.choice([x for x in list(community_map.keys()) if x!=node_map[sample]])])
            else:
                z = [node_map[x] for x in U].index(node_map[sample])
                sample = np.random.choice(U[:z]+U[z+1:])
        else:
            neighbors = list(nx.neighbors(G,sample))
            sample = np.random.choice(neighbors)

        deg_pr_sent = G.degree(sample)
        est_RW_t1 += node_fn(sample)/deg_pr_sent
        est_RW_t2 += 1/deg_pr_sent
        est_RW.append(est_RW_t1/est_RW_t2)
    return np.array(est_RW)

def MHRR_sampling(G,B,U=None):
    #M = np.power(B,2/3)
    #restart_ind = np.power(np.arange(1,M),1.5)
    restart_ind = [10]
    t=20
    while restart_ind[-1]<B:
        restart_ind.append(restart_ind[-1]+int(K*np.log(t)))
        t+=1

    est_MH= []
    est_MH_t = 0
    sample = np.random.choice(G.nodes())
    est_MH_t += node_fn(sample)
    est_MH.append(est_MH_t)
    for ii in range(2,B+1):
        if ii in restart_ind:
            #sample_t = np.random.choice(community_map[np.random.randint(len(community_map))])
            if U is None:
                sample = np.random.choice(community_map[np.random.choice([x for x in list(community_map.keys()) if x!=node_map[sample]])])
            else:
                z = [node_map[x] for x in U].index(node_map[sample])
                sample = np.random.choice(U[:z]+U[z+1:])
        else:
            neighbors = list(nx.neighbors(G,sample))
            sample_t = np.random.choice(neighbors)
            if np.random.rand() <= (G.degree(sample)/G.degree(sample_t)):
                sample = sample_t

        est_MH_t += node_fn(sample)
        est_MH.append(est_MH_t/ii)
    return np.array(est_MH)
```

## F(v) = int(node_map[node] == 1)

```python
def node_fn(node):
    return int(node_map[node]==1)
F_org = sum([node_fn(i) for i in G.nodes()])/G_no_nodes
print(F_org)
```

    0.054015969938938466

```python
MSE_MH_t = 0
for ii in range(1,no_runs+1):
    MSE_MH_t += (MH_sampling(G,max_B)-F_org)**2
MSE_MH = MSE_MH_t/(no_runs)
MSE_MH = np.sqrt(MSE_MH)/F_org

MSE_rds_t = 0
for ii in range(1,no_runs+1):
    MSE_rds_t += (RDS_sampling(G,max_B)-F_org)**2
MSE_rds = MSE_rds_t/(no_runs)
MSE_rds = np.sqrt(MSE_rds)/F_org

MSE_rdsrr_t = 0
for ii in range(1,no_runs+1):
    MSE_rdsrr_t += (RDSRR_sampling(G,max_B)-F_org)**2
MSE_rdsrr = MSE_rdsrr_t/(no_runs)
MSE_rdsrr = np.sqrt(MSE_rdsrr)/F_org

MSE_mhrr_t = 0
for ii in range(1,no_runs+1):
    MSE_mhrr_t += (MHRR_sampling(G,max_B)-F_org)**2
MSE_mhrr = MSE_mhrr_t/(no_runs)
MSE_mhrr = np.sqrt(MSE_mhrr)/F_org

plt.figure(figsize=(10,8))
plt.plot(np.array(list(range(len(MSE_MH)))),MSE_MH,color='red',linewidth=1.5,label='MH')
plt.plot(np.array(list(range(len(MSE_mhrr)))),MSE_mhrr,color='blue',linewidth=1.5,label='MHRR')
plt.plot(np.array(list(range(len(MSE_rds)))),MSE_rds,color='black',linewidth=1.5,label='RDS')
plt.plot(np.array(list(range(len(MSE_rdsrr)))),MSE_rdsrr,color='purple',linewidth=1.5,label='RDSRR')
plt.ylim(top=2.5)

legend = plt.legend(loc='best', shadow=True, fontsize='xx-large')
legend.get_frame().set_facecolor('0.90')
for legobj in legend.legendHandles:
    legobj.set_linewidth(2.5)

plt.grid()
```

## F(v) = int(G.degree(node)>10)

```python
def node_fn(node):
    return int(G.degree(node)>10)
F_org = sum([node_fn(i) for i in G.nodes()])/G_no_nodes
print(F_org)
```

    0.03945514325974636

```python
MSE_MH_t = 0
for ii in range(1,no_runs+1):
    MSE_MH_t += (MH_sampling(G,max_B)-F_org)**2
MSE_MH = MSE_MH_t/(no_runs)
MSE_MH = np.sqrt(MSE_MH)/F_org

MSE_rds_t = 0
for ii in range(1,no_runs+1):
    MSE_rds_t += (RDS_sampling(G,max_B)-F_org)**2
MSE_rds = MSE_rds_t/(no_runs)
MSE_rds = np.sqrt(MSE_rds)/F_org

MSE_rdsrr_t = 0
for ii in range(1,no_runs+1):
    MSE_rdsrr_t += (RDSRR_sampling(G,max_B)-F_org)**2
MSE_rdsrr = MSE_rdsrr_t/(no_runs)
MSE_rdsrr = np.sqrt(MSE_rdsrr)/F_org

MSE_mhrr_t = 0
for ii in range(1,no_runs+1):
    MSE_mhrr_t += (MHRR_sampling(G,max_B)-F_org)**2
MSE_mhrr = MSE_mhrr_t/(no_runs)
MSE_mhrr = np.sqrt(MSE_mhrr)/F_org

plt.figure(figsize=(10,8))
plt.plot(np.array(list(range(len(MSE_MH)))),MSE_MH,color='red',linewidth=1.5,label='MH')
plt.plot(np.array(list(range(len(MSE_mhrr)))),MSE_mhrr,color='blue',linewidth=1.5,label='MHRR')
plt.plot(np.array(list(range(len(MSE_rds)))),MSE_rds,color='black',linewidth=1.5,label='RDS')
plt.plot(np.array(list(range(len(MSE_rdsrr)))),MSE_rdsrr,color='purple',linewidth=1.5,label='RDSRR')
plt.ylim(top=1.2)

legend = plt.legend(loc='best', shadow=True, fontsize='xx-large')
legend.get_frame().set_facecolor('0.90')
for legobj in legend.legendHandles:
    legobj.set_linewidth(2.5)

plt.grid()
```

## F(v) = isprime(v)

```python
def node_fn(node):
    return int(sympy.isprime(G.degree(node)))
F_org = sum([node_fn(i) for i in G.nodes()])/G_no_nodes
print(F_org)
```

    0.5364020666979803

```python
MSE_MH_t = 0
for ii in range(1,no_runs+1):
    MSE_MH_t += (MH_sampling(G,max_B)-F_org)**2
MSE_MH = MSE_MH_t/(no_runs)
MSE_MH = np.sqrt(MSE_MH)/F_org

MSE_rds_t = 0
for ii in range(1,no_runs+1):
    MSE_rds_t += (RDS_sampling(G,max_B)-F_org)**2
MSE_rds = MSE_rds_t/(no_runs)
MSE_rds = np.sqrt(MSE_rds)/F_org

MSE_rdsrr_t = 0
for ii in range(1,no_runs+1):
    MSE_rdsrr_t += (RDSRR_sampling(G,max_B)-F_org)**2
MSE_rdsrr = MSE_rdsrr_t/(no_runs)
MSE_rdsrr = np.sqrt(MSE_rdsrr)/F_org

MSE_mhrr_t = 0
for ii in range(1,no_runs+1):
    MSE_mhrr_t += (MHRR_sampling(G,max_B)-F_org)**2
MSE_mhrr = MSE_mhrr_t/(no_runs)
MSE_mhrr = np.sqrt(MSE_mhrr)/F_org

plt.figure(figsize=(10,8))
plt.plot(np.array(list(range(len(MSE_MH)))),MSE_MH,color='red',linewidth=1.5,label='MH')
plt.plot(np.array(list(range(len(MSE_mhrr)))),MSE_mhrr,color='blue',linewidth=1.5,label='MHRR')
plt.plot(np.array(list(range(len(MSE_rds)))),MSE_rds,color='black',linewidth=1.5,label='RDS')
plt.plot(np.array(list(range(len(MSE_rdsrr)))),MSE_rdsrr,color='purple',linewidth=1.5,label='RDSRR')
plt.ylim(top=0.3)

legend = plt.legend(loc='best', shadow=True, fontsize='xx-large')
legend.get_frame().set_facecolor('0.90')
for legobj in legend.legendHandles:
    legobj.set_linewidth(2.5)

plt.grid()
```

## F(v) = (R(v)+1) x (1+Node_map[v])

```python
def node_fn(node):
    return (1+G.node[node]['classes'])*(1+node_map[node])

F_org = sum([node_fn(i) for i in G.nodes()])/G_no_nodes
print(F_org)
```

    56.59981211836543

```python
MSE_MH_t = 0
for ii in range(1,no_runs+1):
    MSE_MH_t += (MH_sampling(G,max_B)-F_org)**2
MSE_MH = MSE_MH_t/(no_runs)
MSE_MH = np.sqrt(MSE_MH)/F_org

MSE_rds_t = 0
for ii in range(1,no_runs+1):
    MSE_rds_t += (RDS_sampling(G,max_B)-F_org)**2
MSE_rds = MSE_rds_t/(no_runs)
MSE_rds = np.sqrt(MSE_rds)/F_org

MSE_rdsrr_t = 0
for ii in range(1,no_runs+1):
    MSE_rdsrr_t += (RDSRR_sampling(G,max_B)-F_org)**2
MSE_rdsrr = MSE_rdsrr_t/(no_runs)
MSE_rdsrr = np.sqrt(MSE_rdsrr)/F_org

MSE_mhrr_t = 0
for ii in range(1,no_runs+1):
    MSE_mhrr_t += (MHRR_sampling(G,max_B)-F_org)**2
MSE_mhrr = MSE_mhrr_t/(no_runs)
MSE_mhrr = np.sqrt(MSE_mhrr)/F_org

plt.figure(figsize=(10,8))
plt.plot(np.array(list(range(len(MSE_MH)))),MSE_MH,color='red',linewidth=1.5,label='MH')
plt.plot(np.array(list(range(len(MSE_mhrr)))),MSE_mhrr,color='blue',linewidth=1.5,label='MHRR')
plt.plot(np.array(list(range(len(MSE_rds)))),MSE_rds,color='black',linewidth=1.5,label='RDS')
plt.plot(np.array(list(range(len(MSE_rdsrr)))),MSE_rdsrr,color='purple',linewidth=1.5,label='RDSRR')
plt.ylim(top=0.7)

legend = plt.legend(loc='best', shadow=True, fontsize='xx-large')
legend.get_frame().set_facecolor('0.90')
for legobj in legend.legendHandles:
    legobj.set_linewidth(2.5)

plt.grid()
```

## F(v) = random(v)

```python
fn_mapping = np.random.exponential(1,size=(G_no_nodes))
def node_fn(node):
    return fn_mapping[G.nodes().index(node)]

F_org = sum([node_fn(i) for i in G.nodes()])/G_no_nodes
print(F_org)
```

    0.9954489140985855

```python
MSE_MH_t = 0
for ii in range(1,no_runs+1):
    MSE_MH_t += (MH_sampling(G,max_B)-F_org)**2
MSE_MH = MSE_MH_t/(no_runs)
MSE_MH = np.sqrt(MSE_MH)/F_org

MSE_rds_t = 0
for ii in range(1,no_runs+1):
    MSE_rds_t += (RDS_sampling(G,max_B)-F_org)**2
MSE_rds = MSE_rds_t/(no_runs)
MSE_rds = np.sqrt(MSE_rds)/F_org

MSE_rdsrr_t = 0
for ii in range(1,no_runs+1):
    MSE_rdsrr_t += (RDSRR_sampling(G,max_B)-F_org)**2
MSE_rdsrr = MSE_rdsrr_t/(no_runs)
MSE_rdsrr = np.sqrt(MSE_rdsrr)/F_org

MSE_mhrr_t = 0
for ii in range(1,no_runs+1):
    MSE_mhrr_t += (MHRR_sampling(G,max_B)-F_org)**2
MSE_mhrr = MSE_mhrr_t/(no_runs)
MSE_mhrr = np.sqrt(MSE_mhrr)/F_org

plt.figure(figsize=(10,8))
plt.plot(np.array(list(range(len(MSE_MH)))),MSE_MH,color='red',linewidth=1.5,label='MH')
plt.plot(np.array(list(range(len(MSE_mhrr)))),MSE_mhrr,color='blue',linewidth=1.5,label='MHRR')
plt.plot(np.array(list(range(len(MSE_rds)))),MSE_rds,color='black',linewidth=1.5,label='RDS')
plt.plot(np.array(list(range(len(MSE_rdsrr)))),MSE_rdsrr,color='purple',linewidth=1.5,label='RDSRR')

legend = plt.legend(loc='best', shadow=True, fontsize='xx-large')
legend.get_frame().set_facecolor('0.90')
for legobj in legend.legendHandles:
    legobj.set_linewidth(2.5)

plt.ylim(top = 0.3)
plt.grid()
```


# Lecture 4: What are filters?
Audio Processing, MED4, Aalborg University, 2021

By Jesper Kjær Nielsen (jkn@create.aau.dk) and Cumhur Erkut (cer@create.aau.dk)

Last edited: 2021-02-27

```
%matplotlib inline
import numpy as np
import matplotlib.pyplot as plt
import scipy.signal as sig
```

Central aspects of the course
- [x] What is sound?
- [x] How is sound generated?
- [x] How is sound turned into signals (i.e., data) on a computer?
- [ ] **How can we analyse these signals (i.e. extract information from them)?**
- [ ] How can we modify these signals?

Table of Contents

1  Lecture 4: What are filters?

1.1  Introduction to filtering

1.2  Feedforward filters and their impulse responses

1.3  Frequency response of feedforward filters

1.4  Introduction to the feedback filter

## Introduction to filtering
In the next 20 minutes, you will learn
- that you are already using filters in your everyday life
- how you can understand filtering in connection with audio

### Some filters from our everyday life:

#### What about filters for audio?

#### Example: measuring speed using filtering

- Assume that we wish to show the **instantaneous speed** of a car to its driver:
- Unfortunately, we cannot measure the speed directly, but only the driven distance $s(t)$ every $T_\text{s}$ seconds.
- After sampling, the discrete-time signal describing the the driven distance is $s_n=s(t_n)$.

To compute the **instantaneous speed** $v_n$ from $s_n$, we use that the speed of an object is **how much the object moves per unit time**. Thus, we could compute the **instantaneous speed** as
$$
    v_n = \frac{s_n-s_{n-1}}{T_\text{s}} = f_\text{s}(s_n-s_{n-1}) = f_\text{s}s_n-f_\text{s}s_{n-1}\ .
$$

The computation of the instantaneous speed can be interpreted as a **filter** given by
$$
    v_n = f_\text{s}s_n-f_\text{s}s_{n-1}
$$
and this is a so-called **difference equation**.
- Filter input: the discrete-time signal $s_n$ (position)
- Filter output: the discrete-time signal $v_n$ (speed)
- Filtering operation: a function which maps the input signal into the output signal

If we loook at the difference equation
$$
    v_n = f_\text{s}s_n-f_\text{s}s_{n-1}\ ,
$$
we have to do three things to do filtering:
1. **multiply** numbers (e.g., $ax_n$)
2. **sum** numbers (e.g., $x_n+s_n$)
3. **delay** samples (e.g., $x_{n-m}$)

We can also illustrate these three operations graphically.

The filter
$$
    v_n = f_\text{s}s_n-f_\text{s}s_{n-1}\ ,
$$
can then be illustrated as

### A digital filter
We have the following components in a digital filter:
- **Filter input**: a discrete-time signal $x_n$
- **Filter output**: a discrete-time signal $y_n$
- **Filter operation**: a function $f(y_{n-1}, y_{n-2}, \ldots, y_{n-p}, x_n, x_{n-1}, x_{n-2}, \ldots, x_{n-q})$ which computes the current output sample $y_n$ from
 - the current input sample $x_n$ and $q$ past input samples $x_{n-1}, x_{n-2}, \ldots, x_{n-q}$ and
 - the past $p$ output samples $y_{n-1}, y_{n-2}, \ldots, y_{n-p}$

---
Note that the filter output can, in principle, also depend on future values of the input. However, we will not cover this here.

Two main types of filters:
- **Feedforward filter**: the filter output only depends on the filter inputs, i.e.,
$$
    y_n = f(x_n, x_{n-1}, x_{n-2}, \ldots, x_{n-q})
$$
 - The number of past output samples $q$ is zero
 - The feedforward filter is also often referred to a finite impulse response (FIR) filter. More on why later.

- **Feedback filter**: the filter can depend on both the input and output signals, i.e.,
$$
    y_n = f(y_{n-1}, y_{n-2}, \ldots, y_{n-p}, x_n, x_{n-1}, x_{n-2}, \ldots, x_{n-q})
$$
 - The number of past output samples $p$ is at least 1
 - The feedback filter is also often referred to an infinite impulse response (IIR) filter. More on why later.

#### Example: measuring speed using filtering
We can say the following about the filter
$$
    v_n = f_\text{s}s_n-f_\text{s}s_{n-1}.
$$

- The filter input is the signal $s_n$
- The filter output is the signal $v_n$
- The filter is a feedforward filter since the output only depends on the input signal. Thus, $p=0$.
- The order $q$ is one, i.e., the filter is a first-order FIR filter

### Summary
1. Filtering can be based on
 - object size: e.g., coffee and fish
 - wavelength/**frequency**: e.g., light and **audio**
 - space: e.g., microphone array
2. Generally, you can think of a filter as something which extracts what you are interested in.

3. We usually draw a filter as in the figure below where
 - **Filter input**: a discrete-time signal $x_n$
 - **Filter output**: a discrete-time signal $y_n$
 - **Filter operation**: computes the current output sample as a combination of the **current and past input samples** and the **past output samples** (we have not seen the latter yet, though)

### Active break
1. Discuss with your neighbour what the three blocks illustrated below do. What eactly means $x_n$ (discussed in the last lecture) and does it relate to sampling?

Assume that we have a filter given by
$$
    y_n = x_n + b_1 x_{n-1}\ .
$$
2. Illustrate this filter graphically.
3. Let us assume that $x_n = 1$ for $n=0$ and zero otherwise. Compute $y_n$ by hand for $n = 0,1,\ldots,4$.

Now assume that we have the slightly more complicated filter
$$
    y_n = b_0 x_n+b_1 x_{n-1}+b_2x_{n-2}\ .
$$
4. Repeat 2. and 3. from above with this new filter.

## Feedforward filters and their impulse responses
In the next 20 minutes, you will learn
- what a feedforward filter is
- what the impulse response of a filter is

### Difference equation
The difference equation of a feedforward filter is
\begin{align}
    y_n &= b_0x_n + b_1 x_{n-1} + \cdots + b_q x_{n-q}\\
    &= \sum_{m=0}^{q} b_m x_{n-m}
\end{align}
where
- $x_n$ is the input signal
- $y_n$ is the output signal
- $b_0, b_1, \ldots, b_q$ are **filter coefficients**
- we can refer to the past values $x_{n-1}, x_{n-2}, \ldots, x_{n-q}$ as the **filter state**.

### Impulse response of a feedforward filter
#### Example: a simple filter
Let us look at the filter
$$
    y_n = x_n+b_1 x_{n-1}\ .
$$
Thus, we have
- $q=1$
- $b_0=1$
- the filter state is $x_{n-1}$

Let us try to send an impulse
$$
    \delta_n = \begin{cases}
        1 & n=0\\
        0 & \text{otherwise}
    \end{cases}
$$
through the filter.

When we set $x_n=\delta_n$, we get
$$
    y_n = \begin{cases}
        1 & n=0\\
        b_1 & n=1\\
        0 & \text{otherwise}
    \end{cases}\ .
$$

The filter's response to an impulse is referred to as the filter's **impulse response** and usually denoted by $h_n$, i.e., $y_n=h_n$ when $x_n=\delta_n$.

#### Example: a slightly more complicated filter
Let us look at the filter
$$
    y_n = b_0 x_n+b_1 x_{n-1}+b_3 x_{n-3}\ .
$$
Thus, we have
- $q=3$
- $b_2=0$
- the filter state is $x_{n-1}, x_{n-2}, x_{n-3}$

Let us try to send an impulse
$$
    \delta_n = \begin{cases}
        1 & n=0\\
        0 & \text{otherwise}
    \end{cases}
$$
through the filter.

When we set $x_n=\delta_n$, we obtain the impulse response $h_n=y_n$ to
$$
    h_n = \begin{cases}
        b_0 & n=0\\
        b_1 & n=1\\
        b_3 & n=3\\
        0 & \text{otherwise}
    \end{cases}\ .
$$

#### Impulse response of a general feedforward filter
If an impulse $\delta_n$ is send through a feedforward filter of the form
\begin{align}
    y_n &= b_0x_n + b_1 x_{n-1} + \cdots + b_p x_{n-q}\\
    &= \sum_{m=0}^{q} b_m x_{n-m}\ ,
\end{align}
we obtain the impulse response $h_n = y_n$ as
$$
    h_n = \begin{cases}
        b_n & 0\leq n \leq q\\
        0 & \text{otherwise}
    \end{cases}\ .
$$

Since the filter output goes to and stays at 0 for sampling indices $n>q$, the feedforward filter is usually referred to as the **finite impulse response (FIR)** filter.

```
def firFiltering(filterCoefficients, inputSignal, filterState=None):
    nData = np.size(inputSignal)
    firFilterOrder = np.size(filterCoefficients)-1
    # set the filter state to zeros if not specified
    if filterState is None:
        filterState = np.zeros(firFilterOrder)
    # do the filtering
    outputSignal = np.zeros(nData)
    for n in np.arange(nData):
        outputSignal[n] = filterCoefficients[0]*inputSignal[n]
        for m in np.arange(firFilterOrder):
            outputSignal[n] = outputSignal[n] + \
                filterCoefficients[m+1]*filterState[m]
        filterState = np.r_[inputSignal[n],filterState[0:firFilterOrder-1]]
    return outputSignal, filterState
```

```
# compute the impulse response
nData = 10
impulse = np.r_[1, np.zeros(nData-1)]
samplingIndices = np.arange(nData)
filterCoefficients = np.array([1.1, -0.5, 0, 0.5, 0.25])
impulseResponse, filterState = firFiltering(filterCoefficients, impulse)
```

```
# plot the results
plt.figure(figsize=(10,6))
plt.stem(samplingIndices, impulse,'b', markerfmt='bo', label="$\delta_n$", use_line_collection=True)
plt.stem(samplingIndices, impulseResponse, 'r', markerfmt='rx', label="$h_n$", use_line_collection=True)
plt.legend()
plt.xlim((samplingIndices[0]-0.1,samplingIndices[nData-1])), plt.ylim((-0.5,1.1))
plt.xlabel('Sampling index [.]'), plt.ylabel('Signal value [.]');
```

### Summary
1. A feedforward filter has the **difference equation**
\begin{align}
    y_n &= b_0x_n + b_1 x_{n-1} + \cdots + b_p x_{n-q}= \sum_{m=0}^{q} b_m x_{n-m}\ .
\end{align}
2. If we send an impulse $\delta_n$ through a filter, we obtain the filter's **impulse response**.
3. For a general feedforward filter, the impulse response is **finite** and given by
$$
    h_n = \begin{cases}
        b_n & 0\leq n \leq q\\
        0 & \text{otherwise}
    \end{cases}\ .
$$

### Active break
1. Describe in words what an impulse response is and how you can compute it.
2. Sketch the impulse response of the following feedforward filter
$$
    y_n = \frac{1}{q+1}\sum_{m=0}^q x_{n-m}\ .
$$
What are the filter coefficients $b_0, b_1, \ldots, b_q$?
3. Simulate samples for $n=0,1,\ldots,47999$ from the following input signal
$$
    x_n = \cos(2\pi f_1 n/f_\text{s}) + \cos(2\pi f_2 n/f_\text{s})
$$
where $f_\text{s} = 48$ kHz, $f_1 = 500$ Hz, and $f_1 = 5$ kHz. (Tip: use the function `genSinusoid` from the last block of lecture 2). Plot the first 200 samples of the signal $x_n$.
4. Compute $y_n$ by filtering $x_n$ through the filter above (Tip: you can use [scipy's `lfilter` function](https://docs.scipy.org/doc/scipy/reference/generated/scipy.signal.lfilter.html#scipy.signal.lfilter) instead of mine from above.
5. Plot the first 200 samples of $x_n$ and $y_n$ in the same plot. What does the filter seem to do? Can you also hear this if you play back the signals?

## Frequency response of feedforward filters
In the next 20 minutes, you will learn
- what the frequency response of a filter is
- what the frequency response of a feedforward filter is
- what the amplitude and phase response of a filter is

#### Example: a first order FIR filter
Let us again look at the simple filter
$$
    y_n = x_n+b_1 x_{n-1}
$$
which we also looked at before and now know is a **first order FIR filter** with impulse response
$$
    h_n = \begin{cases}
        1 & n=0\\
        b_1 & n=1\\
        0 & \text{otherwise}
    \end{cases}\ .
$$

Instead of an impulse, let us now try to send the phasor $\mathrm{e}^{j\omega n}$ through the filter.

We set $x_n=\mathrm{e}^{j\omega n}$ and get
\begin{align}
    y_n &= \mathrm{e}^{j\omega n} + b_1\mathrm{e}^{j\omega (n-1)} = \mathrm{e}^{j\omega n} + b_1\mathrm{e}^{-j\omega }\mathrm{e}^{j\omega n}\\
    &= \left[1+b_1\mathrm{e}^{-j\omega }\right] \mathrm{e}^{j\omega n}\ .
\end{align}
Note that
- we used the rule $a^n a^m = a^{n+m}$
- the filter output is simply the real value, i.e., $\text{Re}(y_n)$, if we set $x_n=\text{Re}(\mathrm{e}^{j\omega n})=\cos(\omega n)$.

Thus, the output signal is simply the phasor (input signal $x_n$) multiplied by the complex number
$$
    H(\omega) = 1+b_1\mathrm{e}^{-j\omega }
$$
which depends on the digital frequency $\omega$ and filter coefficients $1$ and $b_1$, but not on the sampling index $n$.

When viewed as a function of the digital frequency $\omega$, we call this complex number for the filter's **frequency response**.

Using Euler's formula
$$
    \mathrm{e}^{j\theta} = \cos(\theta)+j\sin(\theta)\ ,
$$
we can write the frequency response $H(\omega)$ in a rectangular form as
$$
    H(\omega) = 1+b_1\cos(\omega)-jb_1\sin(\omega)
$$
which has $1+b_1\cos(\omega)$ as the real part and $-b_1\sin(\omega)$ as the imaginary part.

More interestingly, the polar form of the frequency response can be written as
$$
    H(\omega) = |H(\omega)|\mathrm{e}^{j\angle H(\omega)}
$$
where
\begin{align}
    |H(\omega)| &= \sqrt{(1+b_1\cos(\omega))^2+(-b_1\sin(\omega))^2} = \sqrt{1+b_1^2+2b_1\cos(\omega)}\\
    \angle H(\omega) &= \mathrm{arctan}\left(\frac{-b_1\sin(\omega)}{1+b_1\cos(\omega)}\right)\ .
\end{align}
Note that $|H(\omega)|$ and $\angle H(\omega)$ are usually referred to as the **amplitude response** and **phase response**, respectively.

```
def firFreqResp(filterCoefficients, freqGrid):
    nFreqs = np.size(freqGrid)
    firFilterOrder = np.size(filterCoefficients)-1
    # compute the frequency response
    freqResp = filterCoefficients[0]*np.ones(nFreqs)
    for m in np.arange(firFilterOrder)+1:
        freqResp = freqResp + filterCoefficients[m]*np.exp(-1j*m*freqGrid)
    return freqResp
```

```
# compute the frequency response
nFreqs = 100
freqGrid = 2*np.pi*np.arange(nFreqs)/nFreqs
firFilterCoefficients = np.array([1, 0.9])
_, freqResp = sig.dfreqresp((firFilterCoefficients, 1, 1), freqGrid)
ampResp = np.abs(freqResp)
phaseResp = np.angle(freqResp)
```

```
# plot the results
plt.figure(figsize=(10,6))
plt.subplot(2,1,1)
plt.plot(freqGrid, ampResp, linewidth=2)
plt.xlim((freqGrid[0],freqGrid[nFreqs-1])), plt.ylim((0,1.1*np.max(ampResp)))
plt.xlabel('Dig. freq. [rad/sample]'), plt.ylabel('Amplitude response [.]');
plt.subplot(2,1,2)
plt.plot(freqGrid, phaseResp, linewidth=2)
plt.xlim((freqGrid[0],freqGrid[nFreqs-1])), plt.ylim((-np.pi,np.pi))
plt.xlabel('Dig. freq. [rad/sample]'), plt.ylabel('Phase response [.]');
```

#### Example: filtering two phasors
Assume that we have the signal
$$
    x_n = \mathrm{e}^{j\omega_1 n} + \mathrm{e}^{j\omega_2 n}
$$
where $\omega_1=2\pi f_1/f_\text{s}$ and $\omega_2=2\pi f_2/f_\text{s}$ are **digital frequencies** (see last lecture).

The filter output is then
\begin{align}
    y_n &= x_n+b_1 x_{n-1} = H(\omega_1)\mathrm{e}^{j\omega_1 n}+H(\omega_2)\mathrm{e}^{j\omega_2 n}\\
    &= A_1 \mathrm{e}^{j\omega_1 n+\Phi_1}+ A_2 \mathrm{e}^{j\omega_2 n+\Phi_2}
\end{align}
where the amplitudes and phases of the sinusoids for $i=\{1,2\}$ are given by
\begin{align}
    A_i &= |H(\omega_i)| = \sqrt{1+b_1^2+2b_1\cos(\omega_i)}\\
    \Phi_i &= \angle H(\omega_i) = \mathrm{arctan}\left(\frac{-b_1\sin(\omega_i)}{1+b_1\cos(\omega_i)}\right)\ .
\end{align}

Thus, the filter changes the **amplitudes** and **phases** of the sinusoids, but not the **frequencies**!

### Frequency response of a general feedforward filter
If we send a phasor $\mathrm{e}^{j\omega n}$ through the feedforward filter
\begin{align}
    y_n &= b_0x_n + b_1 x_{n-1} + \cdots + b_q x_{n-q} = \sum_{m=0}^{q} b_m x_{n-m}\ ,
\end{align}
we obtain
$$
    y_n = \sum_{m=0}^{q} b_m \mathrm{e}^{j\omega (n-m)} = \mathrm{e}^{j\omega n} \sum_{m=0}^{q} b_m \mathrm{e}^{-j\omega m}\ .
$$

Thus, the **frequency response** of the feedforward filter is
$$
    H(\omega) = b_0 + b_1\mathrm{e}^{-j\omega} + \cdots + b_q \mathrm{e}^{-j\omega q} = \sum_{m=0}^{q} b_m \mathrm{e}^{-j\omega m}\ .
$$

### The transfer function
When we derived the frequency response, we used a phasor as the input signal, i.e., $x_n=\mathrm{e}^{j\omega n}$. If we set $x_n=z^n$ as the input signal, where $z$ in an arbitrary complex number, we get
$$
    y_n = \sum_{m=0}^{q} b_m z^{n-m} = z^n \sum_{m=0}^{q} b_m z^{-m}\ .
$$
Analogously to the frequency response, we see that the output signal is a multiplication between the input signal and a complex number $H(z)$ given by
$$
    H(z) = b_0 + b_1z^{-1} + \cdots + b_q z^{-q} = \sum_{m=0}^{q} b_m z^{-m}\ .
$$
This complex number is referred to as the filter's **transfer function**.

If we write the complex number $z$ in its polar form $z=r\mathrm{e}^{j\omega}$, we see that
$$
    z^n = (r\mathrm{e}^{j\omega})^n = r^n\mathrm{e}^{j\omega n}\ .
$$
Thus, the frequency response is a special case of the transfer function for $r=1$!

We often prefer working with the transfer function for a couple of reasons:
1. Simpler notation since we can write $z$ instead of $\mathrm{e}^{j\omega}$.
2. The frequency response is a special case of the transfer function (when we set $r=1$ in the latter).
3. The transfer function allow us to make a more thorough analysis of the filter.

We will say much more on the transfer function in lecture 6.

### Summary
1. If we send a phasor $\mathrm{e}^{j\omega n}$ through a filter, we obtain the filter's **frequency response** at the frequency $\omega$.
2. For a general feedforward filter, the frequency response is
$$
    H(\omega) = \sum_{m=0}^{q} b_m \mathrm{e}^{-j\omega m}
$$
which is just a complex number whose
 - magnitude $|H(\omega)|$ is known as the **amplitude response**
 - phase $\angle H(\omega)$ is known as the **phase response**.

### Active break
Consider the simple filter with difference equation
$$
    y_n = 0.5x_n+0.5x_{n-1}\ .
$$
1. What can you say about this filter (type, filter coefficients,  and order $q$)? Make a drawing of the filter.
2. Compute the filter's impulse response $h_n$.
3. Compute the filter's frequency response $H(\omega)$, amplitude response $|H(\omega)|$, and phase response $\angle H(\omega)$. Feel free to use either a computer or pen and paper.
4. Do exercise 1 in the *Introduction to Audio Processing* book (p. 60). You can use Python instead of Pure Data.

---
Tip: Send an impulse $\delta_n$ and a phasor $\mathrm{e}^{j\omega n}$ through the filter to get the impulse response and the frequency response, respectively.  Moreover, a feedforward filter has the difference equation
\begin{align}
    y_n &= b_0x_n + b_1 x_{n-1} + \cdots + b_q x_{n-q} = \sum_{m=0}^{q} b_m x_{n-m}\ .
\end{align}

## Introduction to the feedback filter
In the next 20 minutes, you will learn
- what a feedback filter is
- what the impulse response and frequency response of a feedback filter is
- what a resonance filter is

### Feedback filter
The difference equation of a feedback filter (with $q=1$) is
\begin{align}
    y_n &= b_0x_n + a_1y_{n-1} + \ldots + a_py_{n-p} = b_0x_n + \sum_{m=0}^{p} a_m y_{n-m}
\end{align}
where
- $x_n$ is the input signal
- $y_n$ is the output signal
- $b_0, a_1, a_2, \ldots, a_p$ are **filter coefficients**
- we often refer to the past values $y_{n-1}, y_{n-2}, \ldots, y_{n-p}$ as the **filter state**.

---
Note that general feedback filters can have $q>1$, but we wait with this general case until lecture 6.

### Impulse response of a feedback filter
#### Example: a simple filter
Let us look at the filter
$$
    y_n = x_n+a_1 y_{n-1}\ .
$$
Thus, we have
- $p=0$ and $q=1$
- $b_0=1$
- the filter state is $y_{n-1}$

Let us try to send an impulse
$$
    \delta_n = \begin{cases}
        1 & n=0\\
        0 & \text{otherwise}
    \end{cases}
$$
through the filter.

When we set $x_n=\delta_n$, we get the impulse response $h_n=y_n$ given by
$$
    h_n = \begin{cases}
        a_1^n & n\geq 0\\
        0 & \text{otherwise}
    \end{cases}\ .
$$
Note that $h_n$ never becones zero for $n\geq 0$. Therefore, feedback filters are referred to as **infinite impulse response** (IIR) filters.
- Unlike an FIR filter, the impulse response of a general IIR filter is not simply linked to the filter coefficients.

```
def iirFiltering(filterCoefficients, inputSignal, filterState=None):
    # NOTE: this is only a special case of an IIR filter where the there is no feedforward part,
    # except for x_n which is unscaled (i.e., p=0 and b_0=1).
    nData = np.size(inputSignal)
    iirFilterOrder = np.size(filterCoefficients)
    # set the filter state to zeros if not specified
    if filterState is None:
        filterState = np.zeros(iirFilterOrder)
    # do the filtering
    outputSignal = np.zeros(nData)
    for n in np.arange(nData):
        outputSignal[n] = inputSignal[n]
        for m in np.arange(iirFilterOrder):
            outputSignal[n] = outputSignal[n] + \
                filterCoefficients[m]*filterState[m]
        filterState = np.r_[outputSignal[n],filterState[0:iirFilterOrder-1]]
    return outputSignal, filterState
```

```
# compute the impulse response
nData = 25
impulse = np.r_[1, np.zeros(nData-1)]
samplingIndices = np.arange(nData)
filterCoefficients = np.array([0.7])
impulseResponse, filterState = iirFiltering(filterCoefficients, impulse)
# plot the results
plt.figure(figsize=(10,6))
plt.stem(samplingIndices, impulse,'b', markerfmt='bo', label="$\delta_n$", use_line_collection=1)
plt.stem(samplingIndices, impulseResponse, 'r', markerfmt='rx', label="$h_n$", use_line_collection=1)
plt.legend()
plt.xlim((samplingIndices[0]-0.1,samplingIndices[nData-1])),
plt.ylim((np.minimum(np.min(impulseResponse),0),np.max(impulseResponse)))
plt.xlabel('Sampling index [.]'), plt.ylabel('Signal value [.]');
```

### Frequency response of a feedback filter
#### Example: a resonance filter
Let us look at the filter
$$
    y_n = b_0x_n+a_1 y_{n-1}+a_2 y_{n-2}\ .
$$
Thus, we have
- $p=2$ and $q=0$
- the filter state is $y_{n-1}, y_{n-2}$

For particular values of the filter coefficients, we obtain a **resonance filter** which we will talk more about later.

Let us now try to send the phasor $\mathrm{e}^{j\omega n}$ through the filter.

We set $x_n=\mathrm{e}^{j\omega n}$ and get
\begin{align}
    y_n &= b_0\mathrm{e}^{j\omega n} +a_1 y_{n-1}+a_2 y_{n-2}
\end{align}
which we can rewrite as
$$
    b_0\mathrm{e}^{j\omega n} = y_n - a_1 y_{n-1} - a_2 y_{n-2}\ .
$$

If we guess that the output signal is a multipication of the input phasor and the frequency response, i.e., $y_n=H(\omega)\mathrm{e}^{j\omega n}$, we obtain
\begin{align}
    b_0\mathrm{e}^{j\omega n} &= H(\omega)\mathrm{e}^{j\omega n} - a_1 H(\omega)\mathrm{e}^{j\omega (n-1)} - a_2 H(\omega)\mathrm{e}^{j\omega (n-2)}\\
    &= (1-a_1\mathrm{e}^{-j\omega }-a_2\mathrm{e}^{-j2\omega})H(\omega)\mathrm{e}^{j\omega n}
\end{align}
from which we get that
$$
    H(\omega) = \frac{b_0}{1-a_1\mathrm{e}^{-j\omega }-a_2\mathrm{e}^{-j2\omega}}\ .
$$
The amplitude response $|H(\omega)|$ and the phase response $\angle H(\omega)$ are defined in exactly the same way as for the feedforward filter.

#### Frequency response of a general feedback filter
A filter with the difference equation
\begin{align}
    y_n &= b_0x_n + a_1y_{n-1} + \ldots + a_py_{n-q} = b_0x_0 + \sum_{m=0}^{p} a_m y_{n-m}
\end{align}
has the frequency response
$$
    H(\omega) = \frac{b_0}{1-\sum_{m=1}^p a_m\mathrm{e}^{-j\omega m}}\ .
$$

```
# compute the frequency response
nFreqs = 100
freqGrid = 2*np.pi*np.arange(nFreqs)/nFreqs
iirFilterCoefficients = np.array([1, -0.8])
_, freqResp = sig.dfreqresp((1, iirFilterCoefficients, 1), freqGrid)
ampResp = np.abs(freqResp)
phaseResp = np.angle(freqResp)
```

```
# plot the results
plt.figure(figsize=(10,6))
plt.subplot(2,1,1)
plt.plot(freqGrid, ampResp, linewidth=2)
plt.xlim((freqGrid[0],freqGrid[nFreqs-1])), plt.ylim((0,1.1*np.max(ampResp)))
plt.xlabel('Dig. freq. [rad/sample]'), plt.ylabel('Amplitude response [.]');
plt.subplot(2,1,2)
plt.plot(freqGrid, phaseResp, linewidth=2)
plt.xlim((freqGrid[0],freqGrid[nFreqs-1])), plt.ylim((-np.pi,np.pi))
plt.xlabel('Dig. freq. [rad/sample]'), plt.ylabel('Phase response [.]');
```

### Resonance filter
Assume that we can write the frequency response of a feedback filter as
$$
    H(\omega) = \frac{b_0}{(1-p_1\mathrm{e}^{-j\omega})(1-p_1^*\mathrm{e}^{-j\omega})}
$$
where $p_1$ is a so-called **pole** which we are going to talk more about in lecture 6.

A pole is a complex number so we can write it in a polar form as
\begin{align}
    p_1 &= R\mathrm{e}^{j\theta}\\
    p_1^* &= R\mathrm{e}^{-j\theta}
\end{align}
where $R$ is the magnitude and $\theta$ an angle.

If we insert these expression for $p_1$ and $p_1^*$ in the frequency response, we obtain
\begin{align}
    H(\omega) &= \frac{b_0}{(1-R\mathrm{e}^{j\theta}\mathrm{e}^{-j\omega})(1-R\mathrm{e}^{-j\theta}\mathrm{e}^{-j\omega})}\\
    &= \frac{b_0}{1-2R\cos(\theta)\mathrm{e}^{-j\omega}+R^2\mathrm{e}^{-j2\omega}}\ .
\end{align}

We now see that the resonance filter has filter coefficients
\begin{align}
    a_1 &= 2R\cos(\theta)\\
    a_2 &= -R^2\ .
\end{align}

We now wish to choose $b_0$, $R$, and $\theta$ so that we get
- the desired center frequency $\omega_\text{c}$
- the desired bandwidth $B_\text{w}$
- the desired gain at the center frequency

Unfortunately, it is a bit complicated to show, but it can be shown that
\begin{align}
    \omega_\text{c} &= \arccos\left(\frac{1+R^2}{2R}\cos(\theta)\right)\ ,\quad\text{if }\cos(\theta)\leq\frac{2R}{1+R^2}\\
    B_\text{w} &\approx 2-2R\\
    b_0 &= (1-R^2)\sin(\theta)\ .
\end{align}

From these equations, we can compute the values of $b_0$, $R$, and $\theta$ as
\begin{align}
    R &\approx \frac{2-B_\text{w}}{2}\\
    \theta &= \arccos\left(\frac{2R}{1+R^2}\cos(\omega_\text{c})\right)\\
    b_0 &= (1-R^2)\sin(\theta)\ .
\end{align}

```
def resonanceFreqResp(centerFreq, bandwidth, freqGrid):
    nFreqs = np.size(freqGrid)
    poleRadius = (2-bandwidth)/2
    poleAngle = np.arccos(2*poleRadius*np.cos(centerFreq)/(1+poleRadius**2))
    gain = (1-poleRadius**2)*np.sin(poleAngle)
    iirCoefficients = np.array([2*poleRadius*np.cos(poleAngle), -poleRadius**2])
    freqResp = gain/(1-2*poleRadius*np.cos(poleAngle)*np.exp(-1j*freqGrid)+poleRadius**2*np.exp(-1j*2*freqGrid))
    return freqResp
```

```
# compute the frequency response
nFreqs = 1000
samplingFreq = 1000;
digFreqGrid = 2*np.pi*np.arange(nFreqs)/nFreqs
freqGrid = digFreqGrid*samplingFreq/(2*np.pi)
centerFreq = 2*np.pi*100/samplingFreq
bandwidth = 2*np.pi*50/samplingFreq
freqResp = resonanceFreqResp(centerFreq, bandwidth, digFreqGrid)
ampResp = np.abs(freqResp)
phaseResp = np.angle(freqResp)
```

```
# plot the results
plt.figure(figsize=(10,6))
plt.subplot(2,1,1)
plt.plot(freqGrid, ampResp, linewidth=2)
plt.xlim((freqGrid[0],freqGrid[nFreqs-1])), plt.ylim((0,1.1*np.max(ampResp)))
plt.xlabel('Frequency [Hz]'), plt.ylabel('Amplitude response [.]');
plt.subplot(2,1,2)
plt.plot(freqGrid, phaseResp, linewidth=2)
plt.xlim((freqGrid[0],freqGrid[nFreqs-1])), plt.ylim((-np.pi,np.pi))
plt.xlabel('Frequency [Hz]'), plt.ylabel('Phase response [.]');
```

### Summary
1. A feedback filter (with no feedforward part, i.e., $q=0$) has the difference equation
\begin{align}
    y_n &= b_0x_n + a_1y_{n-1} + \ldots + a_py_{n-p} = b_0x_n + \sum_{m=0}^{p} a_m y_{n-m}\ .
\end{align}
2. The feedback filter is often referred to as an **infinite impulse response** (IIR) filter since its output never becomes zero for a finite sampling index.
3. The frequency response of an IIR filter (with $q=0$) is
$$
    H(\omega) = \frac{b_0}{1-\sum_{m=1}^p a_m\mathrm{e}^{-j\omega m}}\ .
$$
4. A resonance filter is a special kind of filter with the difference equation
\begin{align}
    y_n &= b_0x_n + 2R\cos(\theta)y_{n-1} - R^2y_{n-2}\ .
\end{align}

### Active break
Do exercise 2-4 in the *Introduction to Audio Processing* book (p. 60). You can use Python instead of Pure Data.


# Explicando K-Means Mediante el Algoritmo de Maximización de la Esperanza

> Ya derivamos la forma general del algoritmo de maximización de la esperanza, e hicimos un ejemplo sencillo. Es hora de ver como se aplica este algoritmo a ejemplos más interesantes.

> Ahora, apliquémoslo para explicar el algoritmo K-Means.

> **Objetivos:**
> - Explicar el algoritmo K-Means usando el algoritmo de maximización de la esperanza.

> **Referencias:**
> - Bayesian Methods for Machine Learning course, HSE University, Coursera.

## 1. Recordando el algoritmo K-Means

**Problema:** dado un conjunto de observaciones $x_1, x_2, \dots, x_N \in \mathbb{R}^d$, se debe particionar las $N$ observaciones en $k$ ($\leq N$) clusters $\{1, 2, \dots, k\}$, de manera que se minimice la suma de distancias al cuadrado (varianza).

```python
# Importamos función para generar datos
from bank_customer_data import generate_bank_customer_data
# Importamos pyplot
from matplotlib import pyplot as plt
# Importamos numpy
import numpy as np
```

```python
# Generamos datos
data = generate_bank_customer_data()
data.head()
```

      income
      debt
      labels

      0
      2.535085
      3.818203
      0.0

      1
      3.639738
      2.458079
      0.0

      2
      2.767865
      6.827019
      1.0

      3
      1.863066
      1.496433
      0.0

      4
      0.054357
      2.060369
      0.0

```python
# Datos
plt.scatter(data["income"], data["debt"], c="gray", alpha=0.6)
plt.xlabel("Ingresos mensuales (x100k MXN)")
plt.ylabel("Deuda (x100k MXN)")
```

```python
# Grupos "reales"
plt.scatter(data["income"], data["debt"], c=data["labels"], cmap="Accent", alpha=0.6)
plt.xlabel("Ingresos mensuales (x100k MXN)")
plt.ylabel("Deuda (x100k MXN)")
```

**Algoritmo:**

1. Inicializar los parámetros $\theta = \{\mu_1, \dots, \mu_k\}$ de manera aleatoria.
2. Repetir hasta la convergencia (hasta que los parámetros no varíen):
   1. Para cada punto calcule el centroide más cercano:
      $$
      c_i = \arg \min_{c} ||x_i - \mu_c||.
      $$

   2. Actualizar centroides:
      $$
      \mu_c = \frac{\sum_{i: c_i = c} x_i}{\sum_{i: c_i = c} 1}
      $$

## 2. Conexión con algoritmo de maximización de la esperanza

Desde una perspectiva probabilística, modelamos el K-Means como un modelo de variable latente:

En efecto, la idea es similar al modelo de mezclas Gaussianas, fijando:

- $\Sigma_c = I$.
- $\pi_c = \frac{1}{k}$.

Con lo anterior, la distribución:

$$
p(x_i | t_i=c, \theta) = \mathcal{N}(x_i | \mu_c, I) = \frac{1}{\sqrt{(2 \pi)^n}} \exp\left(-\frac{1}{2}||x_i - \mu_c||^2\right)
$$

### E-step

Para cada punto, debemos encontrar la distribución variacional que minimiza:

$$
q^{j+1} = \arg \min_{q \in Q}\sum_{i=1}^{N} \mathcal{KL}(q(t_i) || p(t_i|x_i, \theta^j)).
$$

Sin embargo, recordamos que el K-Means es un esquema de hard clustering, que solo acepta que un punto pertenezca a un único cluster.

Es decir las posibles opciones para $q$ es el conjunto $Q$ de funciones $\delta$:

$$
Q = \left\{f: f(t)=\delta(t - c) \right\},
$$

donde

$$
\delta(t - c) = \left\lbrace\begin{array}{ccc}
                1 & \text{si} & t=c\\
                0 & \text{en otro caso}
\end{array}
\right.
$$

```python
def fun_delta(t, c):
    return (t == c) * 1
```

```python
t = np.linspace(0, 5, 101)
y = fun_delta(t, 3)
```

```python
plt.plot(t, y, 'ro', ms=8)
plt.vlines(t, 0, y, colors='r', lw=4)
plt.axhline(y=0, color='k')
plt.show()
```

**Pregunta:**

Supongamos que tenemos la siguiente distribución posterior $p(t|x, \theta)$:

```python
plt.plot([1, 2, 3], [0.15, 0.25, 0.6], 'ro', ms=8, label=r"Distribución posterior $p(t|x, \theta)$")
plt.vlines([1, 2, 3], 0, [0.15, 0.25, 0.6], colors='r', lw=4)
plt.axhline(y=0, color='k')
plt.axhline(y=1, ls='--', color='g')
plt.legend()
plt.show()
```

¿Cuál será la función tipo $\delta$ que más se acerque a esta distribución (de acuerdo a la divergencia $\mathcal{KL}$)?

Tenemos 3 opciones razonables para $q$:

1. $q=\delta(t - 1)$

   \begin{align}
   \mathcal{KL}(q(t) || p(t|x, \theta)) & = \mathbb{E}_{q}\left[\log \frac{q(t)}{p(t|x, \theta)}\right] \\
                                        & = \sum_{c=1}^{3} q(t=c)\log \frac{q(t=c)}{p(t=c|x, \theta)} \\
                                        & = 1 \log \frac{1}{0.15} + 0 + 0 \approx 1.8971
   \end{align}

2. $q=\delta(t - 2)$

   \begin{align}
   \mathcal{KL}(q(t) || p(t|x, \theta)) & = \mathbb{E}_{q}\left[\log \frac{q(t)}{p(t|x, \theta)}\right] \\
                                        & = \sum_{c=1}^{3} q(t=c)\log \frac{q(t=c)}{p(t=c|x, \theta)} \\
                                        & = 0 + 1 \log \frac{1}{0.25} + 0 \approx 1.3863
   \end{align}

3. $q=\delta(t - 3)$

   \begin{align}
   \mathcal{KL}(q(t) || p(t|x, \theta)) & = \mathbb{E}_{q}\left[\log \frac{q(t)}{p(t|x, \theta)}\right] \\
                                        & = \sum_{c=1}^{3} q(t=c)\log \frac{q(t=c)}{p(t=c|x, \theta)} \\
                                        & = 0 + 0 + 1 \log \frac{1}{0.6} \approx 0.5108
   \end{align}

De modo que, como se esperaba, la distribución $\delta$ más cercana a la posterior es:

```python
plt.plot([1, 2, 3], [0.15, 0.25, 0.6], 'ro', ms=8, label=r"Distribución posterior $p(t|x, \theta)$")
plt.plot([1, 2, 3], [0, 0, 1], 'bo', ms=5, label=r"Distribución variacional $q(t)$ más cercana")
plt.vlines([1, 2, 3], 0, [0.15, 0.25, 0.6], colors='r', lw=4)
plt.vlines([1, 2, 3], 0, [0, 0, 1], colors='b', lw=2)
plt.axhline(y=0, color='k')
plt.axhline(y=1, ls='--', color='g')
plt.legend()
plt.show()
```

Es decir, la distribución variacional pondrá toda la probabilidad en el valor que tenga la mayor probabilidad en la distribución posterior.

Matemáticamente:

$$
q^{j+1}(t_i) = \delta(t_i - c_i^{j+1}),
$$

donde

$$
c_i^{j+1} = \arg \max_c p(t_i=c | x_i, \theta^j).
$$

Por otra parte, recordemos que la distribución posterior:

\begin{align}
p(t_i=c | x_i, \theta^j) & = \frac{p(x_i | t_i=c, \theta) p(t_i=c | \theta)}{p(x_i | \theta)} \\
                         & = \frac{1}{Z} \exp\left(-\frac{1}{2}||x_i - \mu_c||^2\right) \pi_c \\
                         & = \frac{1}{kZ} \exp\left(-\frac{1}{2}||x_i - \mu_c||^2\right).
\end{align}

De modo que:

$$
c_i^{j+1} = \arg \max_c p(t_i=c | x_i, \theta^j) = \arg \min_c ||x_i - \mu_c||^2
$$

#### ¡Justo como en K-Means!

### M-step

Debemos actualizar los parámetros $\mu$ de modo que maximicen la verosimilitud conjunta (recordar que la matriz de covarianzas y los pesos están fijos):

$$
\max_{\mu} \sum_{i=1}^{N} \mathbb{E}_{q^{j+1}(t_i)} \left[\log p(x_i, t_i| \mu)\right].
$$

Reutilizando lo que habíamos encontrado para el modelo de mezclas Gaussianas en general:

$$
\mu_c^{j+1} = \frac{\sum_{i=1}^{N} q^{j+1}(t_i=c)x_i}{\sum_{i=1}^{N} q(t_i=c)},
$$

donde

$$
q^{j+1}(t_i) = \delta(t_i - c_i^{j+1}).
$$

De esta manera:

$$
\mu_c^{j+1} = \frac{\sum_{i: c_i^{j+1}=c} x_i}{\sum_{i: c_i^{j+1}=c} 1},
$$

#### ¡Justo como en K-Means!

## 3. En resumen

K-Means es un caso particular del algoritmo de maximización de la esperanza para mezclas Gaussianas:

- $\Sigma_c = I$.
- $\pi_c = \frac{1}{k}$.
- Se consideran distribuciones variacionales tipo $\delta$.

Created with Jupyter by Esteban Jiménez Rodríguez.


```python
# preamble to be able to run notebooks in Jupyter and Colab
try:
    from google.colab import drive
    import sys

    drive.mount('/content/drive')
    notes_home = "/content/drive/My Drive/CSC310/notes/"
    user_home = "/content/drive/My Drive/"

    sys.path.insert(1,notes_home) # let the notebook access the notes folder

except ModuleNotFoundError:
    notes_home = "" # running native Jupyter environment -- notes home is the same as the notebook
    user_home = ""  # under Jupyter we assume the user directory is the same as the notebook
```

# Classification Confidence Intervals

**Observation:** It does not matter how careful we are with our model evaluation techniques, there remains a fundamental uncertainty about the ability of our training data to effectively represent our (possibly infinite) data universe.

This uncertainty reflects into our model evaluation. If our training data is a poor representation of the data universe then the models we construct using it will generalize poorly to the rest of the data universe. If our training data is a good representation of the data universe then we can expect that our model will generalize well.

Here we will deal with this uncertainty using *confidence intervals*. First, let us define confidence intervals formally. Given a model accuracy, *acc*, then the confidence interval is defined as the probability *p that our model accuracy acc* lies between some lower bound *lb* and some upper bound *ub*,

\begin{equation}
Pr(lb ≤ acc ≤ ub) = p
\end{equation}

Paraphrasing this equation with *p = 95%*:

>We are 95% percent sure that our model accuracy is not worse than *lb* and not better than *ub*.

Ultimitely we are interested in the lower and upper bounds of the 95% confidence interval. We can use the following formula to compute the bounds:

\begin{equation}
ub = acc + \sqrt\frac{acc(1-acc)}{n}
\end{equation}

\begin{equation}
lb = acc - \sqrt\frac{acc(1-acc)}{n}
\end{equation}

Here, *n* is the number of observations in the testing dataset used to estimate *acc*. The constant 1.96 is called the *z-score* and expresses the fact that we are computing the 95% confidence interval.

# Classification Confidence Intervals in Python

```python
# cross-validation Iris
import pandas as pd
import numpy as np
np.set_printoptions(formatter={'float_kind':"{:3.2f}".format})
from sklearn import tree
# grab cross validation code
from sklearn.model_selection import cross_val_score

# get data
df = pd.read_csv(notes_home+"assets/iris.csv")
X  = df.drop(['id','Species'],axis=1)
y = df['Species']

# set up the model
model = tree.DecisionTreeClassifier(criterion='entropy', max_depth=2)

# do the 5-fold cross validation
scores = cross_val_score(model, X, y, cv=5)
print("Fold Accuracies: {}".format(scores))
```

    Fold Accuracies: [0.93 0.97 0.90 0.87 1.00]

Let's do a simple example using the function classification_confint.

```python
# compute 95% confidence intervals for classification and regression
# problems

def classification_confint(acc, n):
    '''
    Compute the 95% confidence interval for a classification problem.
      acc -- classification accuracy
      n   -- number of observations used to compute the accuracy
    Returns a tuple (lb,ub)
    '''
    import math
    interval = 1.96*math.sqrt(acc*(1-acc)/n)
    lb = max(0, acc - interval)
    ub = min(1.0, acc + interval)
    return (lb,ub)

def regression_confint(rs_score, n, k):
    '''
    Compute the 95% confidence interval for a regression problem.
      rs_score -- R^2 score
      n        -- number of observations used to compute the R^2 score
      k        -- number of independent variables in dataset
    Returns a tuple (lb,ub)
    Reference:
    https://books.google.com/books?id=gkalyqTMXNEC&pg=PA88#v=onepage&q&f=false
    '''
    import math
    interval = 2*math.sqrt((4*rs_score*(1-rs_score)**2*(n-k-1)**2)/((n**2 - 1)*(n+3)))
    lb = max(0, rs_score - interval)
    ub = min(1.0, rs_score + interval)
    return (lb,ub)
```

```python
observations = 100
acc = .88
lb,ub = classification_confint(acc,observations)
print('Accuracy: {} ({:3.2f},{:3.2f})'.format(acc,lb, ub))
```

    Accuracy: 0.88 (0.82,0.94)

Now, let's do an actual example using the Wisconsin breast cancer dataset. We want to print out the testing accuracy together with it's 95% confidence interval.

```python
import pandas as pd
from sklearn import tree
from sklearn.metrics import accuracy_score
from sklearn.model_selection import train_test_split

# read the data
df = pd.read_csv(notes_home+"assets/wdbc.csv")

# set up the feature matrix and target vector
X  = df.drop(['ID','Diagnosis'],axis=1)
y = df['Diagnosis']

# split the data
X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.8, test_size=0.2, random_state=2)

# set up the tree model object - limit the complexity to put us somewhere in the middle of the graph.
model = tree.DecisionTreeClassifier(criterion='entropy', max_depth=4, random_state=1)

# fit the model on the training set of data
model.fit(X_train, y_train)

# Test results: evaluate the model on the testing set of data
y_test_model = model.predict(X_test)
acc = accuracy_score(y_test, y_test_model)
observations = X_test.shape[0]
lb,up = classification_confint(acc, observations)
print("Accuracy: {:3.2f} ({:3.2f},{:3.2f})".format(acc,lb,ub))
```

    Accuracy: 0.91 (0.86,0.94)

## Regression Confidence Intervals

When performing regression we use the $R^2$ score to examine the quality of our models. Given that we only use a small training dataset for fitting the model compared to the rest of the data universe it is only natural to ask what the 95% confidence interval for this score might be. We have a formula for that -- it is not as straight forward as the confidence interval for classification,

\begin{equation}
lb = R^2 - 2\sqrt\frac{4R^2(1-R^2)(n-k-1)}{(n^2-1)(n+3)}
\end{equation}

\begin{equation}
ub = R^2 + 2\sqrt\frac{4R^2(1-R^2)(n-k-1)}{(n^2-1)(n+3)}
\end{equation}

Here, n is the number of observations in the validation/testing dataset and k is the number of independent variables.

```python
# from assets.confint import regression_confint

rs_score = .75
observations = 100
variables = 4 # independent variables

lb,ub = regression_confint(rs_score, observations, variables)
print("R^2 Score: {:3.2f} ({:3.2f}, {:3.2f})".format(rs_score,lb,ub))
```

    R^2 Score: 0.75 (0.67, 0.83)

Let's look at an actual regression problem and compute the R2 score and it's 95% confidence interval. We will use the cars problem from before.

```python
import numpy as np
import pandas
from sklearn.tree import DecisionTreeRegressor
#from assets.confint import regression_confint

# get our dataset
cars_df = pandas.read_csv(notes_home+"assets/cars.csv")

# build model object
model = DecisionTreeRegressor(max_depth=None)

# fit model
# We have to reshape the values array to make 'fit' happy because
# the array only has a single feature
model.fit(cars_df['speed'].values.reshape(-1,1),cars_df['dist'])

# R^2 score
rs_score = model.score(cars_df['speed'].values.reshape(-1,1),cars_df['dist'])
observations = cars_df.shape[0]
variables = 1
lb,ub = regression_confint(rs_score, observations, variables)

# print out R^2 score with its 95% confidence interval
print("R^2 Score: {:3.2f} ({:3.2f}, {:3.2f})".format(rs_score,lb,ub))
```

    R^2 Score: 0.79 (0.69, 0.89)

## Statistical Significance

Besides giving us an idea of the uncertainty of our model the 95% confidence intervals also have something to say about the significance of scores of different models. That is, if the confidence intervals overlap then the difference in model performance of two different models on the same dataset is not statistically significant.

Consider the following,

```python
#from assets.confint import classification_confint

observations = 100

# first classifier
acc1 = .88
lb1,ub1 = classification_confint(acc1,observations)
print('Accuracy: {} ({:3.2f},{:3.2f})'.format(acc1,lb1, ub1))

# second classifier
acc2 = .92
lb2,ub2 = classification_confint(acc2,observations)
print('Accuracy: {} ({:3.2f},{:3.2f})'.format(acc2,lb2, ub2))
```

    Accuracy: 0.88 (0.82,0.94)
    Accuracy: 0.92 (0.87,0.97)

Even though the second classifier has a better raw accuracy when we look at the confidence intervals of the two classifiers we see that they overlap. Here we see that the first classifier could potentially have an accuracy of .94 (even better than the raw accuracy of the second classifier). Furthermore, the confidence interval of the second classifier tells us that that classifier could potentially have an accuracy of .87 which is worse than the raw accuracy of the first classifier. For this reason we say that the difference in accuracy of two classifiers is not statistically significant if their confidence intervals overlap.


# Advanced Features

When analyzing the real world datasets, we may have the following targets:

1. certain variables must be selected when some prior information is given;
2. selecting the weak signal variables when the prediction performance is mainly interested;
3. identifying predictors when group structure are provided;
4. pre-excluding a part of predictors when datasets have ultra high-dimensional predictors;
5. specify the division of sample in cross validation.

In the following content, we will illustrate the statistic methods to reach these targets in a one-by-one manner, and give quick examples to show how to perform the statistic methods in `abessLm` and the same steps can be implemented in all methods. Actually, in our methods, the targets can be properly handled by simply change some default arguments in the functions.

## Nuisance Regression

Nuisance regression refers to best subset selection with some prior information that some variables are required to stay in the active set. For example, if we are interested in a certain gene and want to find out what other genes are associated with the response when this particular gene shows effect.

In the `abessLm()` (or other methods), the argument `always_select` is designed to realize this goal. User can pass a vector containing the indexes of the target variables to `always_select`. Here is an example.

```python
import numpy as np
from abess.datasets import make_glm_data
from abess.linear import abessLm

np.random.seed(0)

# gene data
n = 100
p = 20
k = 5
dt = make_glm_data(n = n, p = p, k = k, family = 'gaussian')
print('real coefficients:\n', dt.coef_, '\n')
print('real coefficients\' indexes:\n', np.nonzero(dt.coef_)[0])
```

    real coefficients:
     [  0.           0.         115.01218243   0.           0.
      81.84924757   0.           0.           0.           0.
     104.77568224  64.30426355   0.           0.           0.
       0.           0.           0.         108.5408557    0.        ]

    real coefficients' indexes:
     [ 2  5 10 11 18]

```python
model = abessLm(support_size = range(0, 6))
model.fit(dt.x, dt.y)
print('fitted coefficients:\n', model.coef_, '\n')
print('fitted coefficients\' indexes:\n', np.nonzero(model.coef_)[0])
```

    fitted coefficients:
     [  0.           0.         114.93792167   0.           0.
      81.8124385    0.           0.           0.           0.
     104.67076232  64.40856101   0.           0.           0.
       0.           0.           0.         108.73726174   0.        ]

    fitted coefficients' indexes:
     [ 2  5 10 11 18]

The coefficients are located in \[2, 5, 10, 11, 18\].
But if we suppose that the 7th and 8th variables are worthy to be included in the model, we can call:

```python
model = abessLm(support_size = range(0, 6), always_select = [7, 8])
model.fit(dt.x, dt.y)
print('fitted coefficients:\n', model.coef_, '\n')
print('fitted coefficients\' indexes:\n', np.nonzero(model.coef_)[0])
```

    fitted coefficients:
     [  0.           0.         117.18370615   0.           0.
       0.           0.           5.09643891  -1.00521149   0.
      91.65760504   0.           0.           0.           0.
       0.           0.           0.         121.21120638   0.        ]

    fitted coefficients' indexes:
     [ 2  7  8 10 18]

Now the variables we chosen are always in the model.

## Regularized Adaptive Best Subset Selection

In some cases, especially under low signal-to-noise ratio (SNR) setting or predictors are highly correlated, the vallina type of $L_0$ constrained model may not be satisfying and a more sophisticated trade-off between bias and variance is needed. Under this concern, the `abess` pakcage provides option of best subset selection with $L_2$ norm regularization called the regularized bess. The model has this following form:

$$
\begin{align}
    \arg\min_\beta L(\beta) + \alpha \|\beta\|_2^2.
\end{align}
$$

To implement the regularized bess, user need to specify a value to an additive argument `alpha` in the `abessLm()` function (or other methods). This value corresponds to the penalization parameter in the model above.

Let’s test the regularized best subset selection against the no-regularized one over 100 replicas in terms of prediction performance. With argument `snr` in `make_glm_data()`, we can add white noise into generated data.

```python
loss = np.zeros((2, 100))
coef = np.repeat([1, 0], [5, 25])
for i in range(100):
    np.random.seed(i)
    train = make_glm_data(n = 100, p = 30, k = 5, family = 'gaussian', coef_ = coef, snr = 0.05)
    np.random.seed(i + 100)
    test = make_glm_data(n = 100, p = 30, k = 5, family = 'gaussian', coef_ = coef, snr = 0.05)

    # normal
    model = abessLm()
    model.fit(train.x, train.y)
    loss[0, i] = np.linalg.norm(model.predict(test.x) - test.y)
    # regularized
    model = abessLm(alpha = 0.7)
    model.fit(train.x, train.y)
    loss[1, i] = np.linalg.norm(model.predict(test.x) - test.y)

print("normal model's loss:", np.mean(loss[0,:]))
print("regularized model's loss:", np.mean(loss[1,:]))
```

    normal model's loss: 24.709066304317844
    regularized model's loss: 23.416157533057604

The regularized model has a lower test loss. And we can also make a boxplot:

```python
import matplotlib.pyplot as plt
plt.boxplot([loss[0,:], loss[1,:]], labels = ['ABESS', 'RABESS'])
plt.show()
```

We see that the regularized best subset select ("RABESS" in figure)  indeed reduces the prediction error.

## Best group subset selection

Best group subset selection (BGSS) aims to choose a small part of non-overlapping groups to achieve the best interpretability on the response variable. BGSS is practically useful for the analysis of ubiquitously existing variables with certain group structures. For instance, a categorical variable with several levels is often represented by a group of dummy variables. Besides, in a nonparametric additive model, a continuous component can be represented by a set of basis functions (e.g., a linear combination of spline basis functions). Finally, specific prior knowledge can impose group structures on variables. A typical example is that the genes belonging to the same biological pathway can be considered as a group in the genomic data analysis.

The BGSS can be achieved by solving:

$$
    \min_{\beta\in \mathbb{R}^p} \frac{1}{2n} ||y-X\beta||_2^2,\quad s.t.\ ||\beta||_{0,2}\leq s .
$$

where $||\beta||_{0,2} = \sum_{j=1}^J I(||\beta_{G_j}||_2\neq 0)$ in which $||\cdot||_2$ is the $L_2$ norm and model size $s$ is a positive integer to be determined from data. Regardless of the NP-hard of this problem, Zhang et al develop a certifiably polynomial algorithm to solve it. This algorithm is integrated in the `abess` package, and user can handily select best group subset by assigning a proper value to the `group` arguments:

We still use the dataset `dt` generated before, which has 100 samples, 5 useful variables and 15 irrelevant varibales.

```python
print('real coefficients:\n', dt.coef_, '\n')
```

    real coefficients:
     [  0.           0.         115.01218243   0.           0.
      81.84924757   0.           0.           0.           0.
     104.77568224  64.30426355   0.           0.           0.
       0.           0.           0.         108.5408557    0.        ]

Support we have some prior information that every 5 variables as a group:

```python
group = np.linspace(0, 3, 4).repeat(5)
print('group index:\n', group)
```

    group index:
     [0. 0. 0. 0. 0. 1. 1. 1. 1. 1. 2. 2. 2. 2. 2. 3. 3. 3. 3. 3.]

Then we can set the `group` argument in function. Besides, the `support_size` here indicates the number of groups, instead of the number of variables.

```python
model = abessLm(support_size = range(0, 3))
model.fit(dt.x, dt.y, group = group)
print('coefficients:\n', model.coef_)
```

    coefficients:
     [  4.07330876  14.02654966 133.63659942  -3.25926433  -8.02172721
       0.           0.           0.           0.           0.
       0.           0.           0.           0.           0.
      -4.14697258   1.53447211  16.29386214 112.43896075   8.85388996]

The fitted result suggest that only two groups are selected (since `support_size` is from 0 to 2) and the selected variables are shown before.

## Integrate SIS

Ultra-high dimensional predictors increase computational cost but reduce estimation accuracy for any statistical procedure. To reduce dimensionality from high to a relatively acceptable level, a fairly general asymptotic framework, named feature screening (sure independence screening) is proposed to tackle even exponentially growing dimension. The feature screening can theoretically maintain all effective predictors with a high probability, which is called "the sure screening property".

In our program, to carrying out the Integrate SIS, user need to set `is_screening = True` and pass an integer smaller than the number of the predictors to the `screening_size`. Then the program will first calculate the marginal likelihood of each predictor and reserve those predictors with the `screening_size` largest marginal likelihood. Then, the ABESS algorithm is conducted only on this screened subset.

Here is an example.

```python
n = 100
p = 1000
k = 3
np.random.seed(2)

# gene data
dt = make_glm_data(n = n, p = p, k = k, family = 'gaussian')
print('real coefficients\' indexes:', np.nonzero(dt.coef_)[0])

# fit
model = abessLm(support_size = range(0, 5), is_screening = True, screening_size = 100)
model.fit(dt.x, dt.y)
print('fitted coefficients\' indexes:', np.nonzero(model.coef_)[0])
```

    real coefficients' indexes: [243 295 659]
    fitted coefficients' indexes: [243 295 659]

## User-specified cross validation division

Sometimes, especially when running a test, we would like to fix the train and valid data used in cross validation, instead of choosing them randomly.
One simple method is to fix a random seed, such as `numpy.random.seed()`. But in some cases, we would also like to specify which samples would be in the same "fold", which has great flexibility.

In our program, an additional argument `cv_fold_id` is for this user-specified cross validation division. An integer array with the same size of input samples can be given, and those with same integer would be assigned to the same "fold" in K-fold CV.

```python
n = 100
p = 1000
k = 3
np.random.seed(2)

dt = make_glm_data(n = n, p = p, k = k, family = 'gaussian')

# cv_fold_id has a size of `n`
# cv_fold_id has `cv` different integers
cv_fold_id = [1 for i in range(30)] + [2 for i in range(30)] + [3 for i in range(40)]

model = abessLm(support_size = range(0, 5), cv = 3)
model.fit(dt.x, dt.y, cv_fold_id = cv_fold_id)
print('fitted coefficients\' indexes:', np.nonzero(model.coef_)[0])
```

    fitted coefficients' indexes: [243 295 659]

## R tutorial

For R tutorial, please view [https://abess-team.github.io/abess/articles/v07-advancedFeatures.html](https://abess-team.github.io/abess/articles/v07-advancedFeatures.html).


# Supplementary materials : Details on generating Figure 1

This document is a supplementary material of the article *Detecting periodicities with Gaussian
processes* by N. Durrande, J. Hensman, M. Rattray and N. D. Lawrence.

The first step is to import the required packages. This tutorial has been written with GPy 0.8.8 which includes the kernels discussed in the article. The latter can be downloaded on the [SheffieldML github page](https://github.com/SheffieldML/GPy).

```python
%matplotlib inline
import numpy as np
from matplotlib import pyplot as plt
import GPy

np.random.seed(1)
```

## Test functions

We now introduce 1-periodic tests functions that are defined over $[0,1)$ as:
\begin{equation}
	\begin{split}
	f_1(x)	& = \cos(2 \pi x) + \varepsilon\\
	f_2(x)	& = 1/2 \cos(2 \pi x) + 1/2 \cos(4 \pi x) + \varepsilon\\
	f_3(x)	& = \left\{
	\begin{matrix}
		1 + \varepsilon \text{    if } x \in [0,0.2]  \\
		-1 + \varepsilon \text{    if } x \in (0.2,1) \\
	\end{matrix}
	\right. \\
	f_4(x)	& = 4 |x-0.5| + 1) + \varepsilon\\
	f_5(x)	& = 1 - 2x + \varepsilon\\
	f_6(x)	& = + \varepsilon
	\end{split}
\end{equation}
where $\varepsilon$ is a $\mathcal{N}(0,\tau^2)$ random variable.

```python
# domain boundaries
lower = 0.
upper = 3.
x = np.linspace(lower,upper,500)

# grid for function evaluations and plots
n_pts = 50
X = np.linspace(lower,upper,n_pts+1)
X = X[0:-1]+X[1]/2

# test functions
def f1(x,tau2=0.1):
    return(np.cos(x*2*np.pi) + np.sqrt(tau2)*np.random.normal(size=x.shape))

def f2(x,tau2=0.1):
    return(1./2*np.cos(x*2*np.pi)+1./2*np.cos(x*4*np.pi) + np.sqrt(tau2)*np.random.normal(size=x.shape))

def f3(x,tau2=0.1):
    alpha = 0.2
    return(2*(x - np.trunc(x) < alpha)-1 + np.sqrt(tau2)*np.random.normal(size=x.shape))

def f4(x,tau2=0.1):
    return(4*np.abs(x - np.trunc(x) - 0.5)-1 + np.sqrt(tau2)*np.random.normal(size=x.shape))

def f5(x,tau2=0.1):
    return(2*np.trunc(x) - 2*x +1 + np.sqrt(tau2)*np.random.normal(size=x.shape))

def f6(x,tau2=0.1):
    return(np.zeros((len(x),)) +  np.sqrt(tau2)*np.random.normal(size=x.shape))

```

The associated graphs are:

```python
names = ['f1','f2','f3','f4','f5','f6']

fig, axs = plt.subplots(2,3,figsize=(12,7), sharex=True, sharey=True)
for i, (ax,testfunc) in enumerate(zip(axs.flat, [f1,f2,f3,f4,f5,f6])):
    ax.plot(x,testfunc(x,0.),'r', linewidth=1.5)
    ax.plot(X,testfunc(X),'kx',mew=1)
    ax.legend(['$\\mathrm{'+names[i]+'}$'],prop={'size':18},borderaxespad=0.)
    ax.set_ylim((-1.5,1.8))
```

## Models

**COSOPT:** We consider here the following implementation

```python
def fit_cosopt(X,Y):
    X = X[:,None]
    Y = Y[:,None]
    period = np.linspace(0.15,2,100)
    phase = np.linspace(-np.pi,np.pi,100)

    MSE = np.zeros((100,100))
    for i,per in enumerate(period):
        for j,pha in enumerate(phase):
            B = np.hstack((np.ones(X.shape),np.cos(X*2*np.pi/per+pha)))
            C = np.dot(np.linalg.inv(np.dot(B.T,B)),np.dot(B.T,Y))
            MSE[i,j] = np.mean((np.dot(B,C)-Y)**2)

    i,j = np.unravel_index(MSE.argmin(), MSE.shape)
    B = np.hstack((np.ones(X.shape),np.cos(X*2*np.pi/period[i]+phase[j])))
    C = np.dot(np.linalg.inv(np.dot(B.T,B)),np.dot(B.T,Y))

    return((C,period[i],phase[j]))

def pred_cosopt(x,m_cosopt):
    C,per,pha = m_cosopt
    Bx = np.hstack((0*x[:,None]+1, np.cos(x[:,None]*2*np.pi/per+pha)))
    P = np.dot(Bx,C)
    return(P.flatten())

```

**Linear regression**

```python
def B(x):
    # function returning the matrix of basis functions evaluated at x
    #input:   x, np.array with d columns
    #output:  a matrix (b_j(x_i))_{i,j}
    B = np.ones((x.shape[0],1))
    for i in range(1,20):
        B = np.hstack((B,np.sin(2*np.pi*i*x[:,None]),np.cos(2*np.pi*i*x[:,None])))
    return(B)

def LR(X,F,B,tau2):
    #input:   X, np.array with d columns representing the DoE
    #         F, np.array with 1 column representing the observations
    #         B, a function returning the (p) basis functions evaluated at x
    #         tau2, noise variance
    #output:  beta, estimate of coefficients np.array of shape (p,1)
    #         covBeta, cov matrix of beta, np.array of shape (p,p)
    BX = B(X)
    covBeta = np.linalg.inv(np.dot(BX.T,BX))
    beta = np.dot(covBeta,np.dot(BX.T,F))
    return(beta,tau2*covBeta)

def predLR(x,B,beta,covBeta):
    #function returning predicted mean and variance
    #input:   x, np.array with d columns representing m prediction points
    #         B, a function returning the (p) basis functions evaluated at x
    #         beta, estimate of the regression coefficients
    #         covBeta, covariance matrix of beta
    #output:  m, predicted mean at x, np.array of shape (m,1)
    #         v, predicted variance, np.array of shape (m,1)
    m = np.dot(B(x),beta)
    v = np.dot(B(x),np.dot(covBeta,B(x).T))
    return(m,v)

```

**Gaussian Process model**

```python
def fit_gp(X,Y):
    #input:   X, np.array with d columns representing the DoE
    #         Y, np.array with 1 column representing the observations
    #output:  a GPy gaussian process model object
    X = X[:,None]
    Y = Y[:,None]
    k = GPy.kern.PeriodicMatern32(1,variance=1.,lengthscale=1., period=1., n_freq=20,lower=lower,upper=upper)
    bias = GPy.kern.Bias(1,variance=1.)
    m32 = GPy.models.GPRegression(X,Y,k+bias)
    m32.unconstrain('')                               # remove positivity constrains to avoids warnings
    m32.likelihood.constrain_bounded(0.001,3., warning=False)        # boundaries for the observation noise
    m32.kern.periodic_Matern32.constrain_bounded(0.01,3., warning=False)     # boundaries for the periodic variance and lengthscale
    m32.kern.periodic_Matern32.period.constrain_bounded(0.15,2., warning=False)        # boundaries for the period
    m32.randomize()
    m32.optimize_restarts(5,robust=True)
    return(m32)

def pred_gp(x,m_gp):
    x = x[:,None]
    mu,var = m_gp.predict(x)
    return(mu.flatten())

```

Definition of the criterion for assesing the quality of a model prediction :

```python
def RMSE(Ypred,Yreal):
    return( np.sqrt(np.mean((Yreal - Ypred)**2)) )
```

## Fit models
The following loop will fit each model for each test function (takes about 30 seconds) :

```python
M_COS = []
M_GP  = []
M_LR  = []

for i,testfunc in enumerate([f1,f2,f3,f4,f5,f6]):
    Y = testfunc(X)
    Yreal  = testfunc(x,0)
    M_COS += [fit_cosopt(X,Y)]
    M_GP  += [fit_gp(X,Y)]
    M_LR  += [LR(X,Y,B,0.1)]
```

    Optimization restart 1/5, f = 64.6381557382
    Optimization restart 2/5, f = 20.1924219702
    Optimization restart 3/5, f = 64.2805490956
    Optimization restart 4/5, f = 64.638206481
    Optimization restart 5/5, f = 64.6381544424
    Optimization restart 1/5, f = 52.8350376662
    Optimization restart 2/5, f = 29.5456094222
    Optimization restart 3/5, f = 29.545330048
    Optimization restart 4/5, f = 45.9666595165
    Optimization restart 5/5, f = 29.5453299967
    Optimization restart 1/5, f = 62.9316629092
    Optimization restart 2/5, f = 43.5915711548
    Optimization restart 3/5, f = 45.4372991694
    Optimization restart 4/5, f = 43.5915711561
    Optimization restart 5/5, f = 43.5915711548
    Optimization restart 1/5, f = 15.6691357445
    Optimization restart 2/5, f = 15.6691357411
    Optimization restart 3/5, f = 15.6691357415
    Optimization restart 4/5, f = 23.0512998344
    Optimization restart 5/5, f = 51.6129398693
    Optimization restart 1/5, f = 31.676011619
    Optimization restart 2/5, f = 46.6728376247
    Optimization restart 3/5, f = 31.611689604
    Optimization restart 4/5, f = 31.6116896036
    Optimization restart 5/5, f = 46.1722924018
    Optimization restart 1/5, f = 10.8131947043
    Optimization restart 2/5, f = 10.2223185861
    Optimization restart 3/5, f = 10.2223189124
    Optimization restart 4/5, f = 10.2223193099
    Optimization restart 5/5, f = 10.7196113476

## Generate figure

```python
lower = 0.
upper = 3.
xRMSE = np.linspace(lower,upper,500)

fig, axes = plt.subplots(2,3,figsize=(12,7), sharex=True, sharey=True, tight_layout=False)
for i,testfunc in enumerate([f1,f2,f3,f4,f5,f6]):
    Y = testfunc(X)
    Yreal = testfunc(x,0)
    ax = axes.flat[i]
    # test func
    plreal, = ax.plot(x,Yreal, '-r', linewidth=1.5,label="test function")
    realRMSE = testfunc(xRMSE,0)
    # COSOPT
    cosopt_pred = pred_cosopt(x,M_COS[i])
    plcos, = ax.plot(x,cosopt_pred, '--g', linewidth=1.5,label="COSOPT", dashes=(7,3))
    cRMSE = RMSE(pred_cosopt(xRMSE,M_COS[i]),realRMSE)
    #GP 32
    gp_pred = pred_gp(x,M_GP[i])
    plgp, = ax.plot(x,gp_pred, '--b', linewidth=1.5,label="periodic GP", dashes=(15,3))
    gpRMSE = RMSE(pred_gp(xRMSE,M_GP[i]),realRMSE)
    # Lin Reg
    lr_pred = predLR(x,B,M_LR[i][0],M_LR[i][1])
    pllr, = ax.plot(x,lr_pred[0], ':k', linewidth=2,label="periodic GP")
    lrRMSE = RMSE(predLR(xRMSE,B,M_LR[i][0],M_LR[i][1])[0],realRMSE)
    #
    ax.plot(X,testfunc(X),'kx',mew=1, alpha=0.5)
    ax.set_xlim((0.9,2.1))
    ax.set_ylim((-1.5,2.))
    ## RMSE
    ax.text(1.5, 1.8, 'RMSE=(%.2f, %.2f, %.2f)'%(cRMSE,gpRMSE,lrRMSE),
        verticalalignment='center', horizontalalignment='center', fontsize=13)

fig.suptitle('  ')
l = fig.legend((plreal,plcos,plgp,pllr),("test function","COSOPT","periodic GP","Lin. Reg."),'upper center',ncol=4,handlelength=3,fancybox=True,columnspacing=3)
l.draw_frame(False)

```


# The Schrödinger Equation

>The underlying physical laws necessary for the mathematical theory of a large part of physics and the whole of chemistry are thus completely known, and the difficulty is only that the exact application of these laws leads to equations much too complicated to be soluble. It therefore becomes desirable that approximate practical methods of applying quantum mechanics should be developed, which can lead to an explanation of the main features of complex atomic systems without too much computation.
    - Paul A. M. Dirac (1929)

## 🥅 Learning Objectives
- Review of Plane Wave(functions)
- "Derivation" of the Time-Dependent Schrödinger Equation (TDSE)
- The Hamiltonian
- The Time-Independent Schrödinger Equation (TISE)
- Separation of Variables Method

## Motivation for the Schrödinger Equation
Just like Newton’s equations or Maxwell’s equations, the Schrödinger equation is a law of nature:  it cannot be “derived.”  Instead, it must be postulated, and then tested by experiment.  However, just as Newton’s equations can be justified insofar as they encapsulate the “laws of motion” formulated by Galileo and Newton, and Maxwell’s equations can be justified insofar as they encapsulate the “laws of electromagnetism” formulated by Gauss, Faraday, Coulomb, etc., Schrödinger's equation can be justified by observing that it encapsulates the “laws” of Planck and De Broglie, namely that

$$ E = h \nu = \hbar \omega $$

$$ p = \tfrac{h}{\lambda} = \tfrac{h \nu}{c} = \hbar k = h \tilde{\nu} $$

### Angular Frequency and Wavenumber
In these equations I have introduced several new symbols, mostly related to the fact it is often convenient to use angular frequency,

$$ \omega = 2 \pi \nu $$

which naturally pairs with

$$ \hbar = \tfrac{h}{2 \pi} $$

Similarly, it is often convenient, especially in spectroscopic studies, to use wavenumber,

$$ \tilde{\nu} = \tfrac{1}{\lambda} = \lambda^{-1} $$

or its angular analogue

$$ k = \tfrac{2 \pi}{\lambda} = 2 \pi \tilde{nu} $$

Sometimes it is also useful to consider the period of the wave,

$$ \text{T}  = \nu^{-1} $$

Recall that the wavelength, frequency, period, and wavenumber of a wave can be related to the speed of light by relations like:

$$ c = \lambda \nu = \tfrac{\lambda}{\text{T}} = \tilde{\nu} \nu $$

$$ c = \tfrac{\omega}{k} $$

#### 📝 Exercise: The photon emitted when a 4p electron in the Hydrogen atom deexcites into a 2s orbital has wavelength 486.1 nm. Compute its
- **frequency in Hz**
- **angular frequency in Hz**
- **wavenumber, $\bar{\nu}$, in $\text{cm}^{-1}$.**
- **angular wavenumber, $k$, in $\text{m}^{-1}$.**
- **period in s.**
- **momentum in m kg/s.**
- **energy in Joules.**
[Answer](SchrodingerExercise1.ipynb)

### The Wavefunction

The use of angular frequency and wave number is especially useful when dealing with waves, because the need to explicitly include factors of $2 \pi$ to describe the periodic oscillations is removed. A one-dimensional plane wave, moving in the $x$ direction, with amplitude $A$, wavelength $\lambda = \tfrac{2 \pi}{k}$, and frequency $\nu = \tfrac{\omega}{2 \pi}$ is described by

$$ \Psi(x,t) = A e^{i (kx - \omega t)} = A\left(\cos(kx-\omega t) + i \sin(kx - \omega t) \right) $$

Since $\Psi(x,t)$ is the function that describes a wave, we call it a "wavefunction."

### Expression for the Total Energy
Based on electromagnetism, we believe this equation should give us the essential insight we need to describe the "waviness of particles." We need, however, a second equation that will allow us to describe the "particulateness of waves." Since the energy of a wave is proportional to $\omega$ (Planck-Einstein relation) and the momentum of a wave is proportional to $k$ (De Broglie relation), an equation that relates the energy to the momentum is an obvious candidate. For example,

$$
\begin{align}
\text{energy} &= \text{kinetic energy} + \text{potential energy}\\
E &= T + V(x,t)\\
E &= \frac{p^2}{2m} + V(x,t)
\end{align}
$$

### The Time-Dependent Schrödinger Equation (TDSE)
We need to find a way to combine the key equations from the previous two sections. To do this, note that

$$ \frac{\partial \Psi(x,t)}{\partial t} = (-i \omega) A e^{i (kx - \omega t)} = (-i \omega) \Psi(x,t) $$

We can link this to Planck's equation for the energy of a photon by multiplying both sides by $i \hbar$,

$$
\begin{align}
i \hbar \frac{\partial \Psi(x,t)}{\partial t} &= (i \hbar) (-i \omega) \Psi(x,t)  = \hbar \omega \Psi(x,t) \\
&= E \Psi(x,t)
\end{align}
$$

Similarly, differentiating with respect to position,

$$
\begin{align}
\frac{\partial \Psi(x,t)}{\partial x} &= (i k) A e^{i (kx - \omega t)} \\
\frac{\partial^2 \Psi(x,t)}{\partial x^2} &= (i k)^2 A e^{i (kx - \omega t)} = -k^2 \Psi(x,t)
\end{align}
$$

Multiplying both sides by $-\hbar^2$ provides a link to the De Broglie relation,

$$
-\hbar^2 \frac{\partial^2 \Psi(x,t)}{\partial x^2} = (\hbar k)^2 \Psi(x,t) = p^2 \Psi(x,t)
$$

Now, let's consider our expression for the energy,

$$
\begin{align}
E &= \frac{p^2}{2m} + V(x,t) \\
E \Psi(x,t) &= \frac{p^2}{2m} \Psi(x,t) + V(x,t)\Psi(x,t)
\end{align}
$$

Substituting in the equations for the energy and the momentum-squared in terms of wavefunction derivatives, we have:

$$ i \hbar \frac{\partial \Psi(x,t)}{\partial t} = -\frac{\hbar^2}{2m}\frac{\partial^2 \Psi(x,t)}{\partial x^2} + V(x,t)\Psi(x,t) $$

This is the [*time-dependent Schrödinger equation*](https://en.wikipedia.org/wiki/Schr%C3%B6dinger_equation#Time-dependent_equation), which we often abbreviate as the TDSE.

The biggest limitations of the Schrödinger equation is that it was derived using the nonrelativistic expression for the kinetic energy of a particle, $T = \tfrac{p^2}{2m}$. For this reason, the Schrödinger equation is not valid for particles that are moving at a substantial fraction of the speed of light (where relativistic effects are important) or for light itself. Quantum mechanics can be extended to such cases, and relativistic quantum mechanics is quite important for the heavier atoms in the periodic table. (Typically relativity is neglected for atoms up to Zinc (Z=30) or Krypton (Z=36), and the included past that. However there are a few molecules containing lighter elements—e.g., the Sulfur dimer—where there are chemically (or at least spectroscopically) interesting relativistic effects.) The full relativistic treatment of quantum mechanics leads to “quantum electrodynamics,” which is a rich and interesting subject that we, alas, will not have time to cover.

#### 📝 Exercise: Show that the time-dependent Schrödinger equation for the complex conjugate of the wavefunction, $\Psi^*$, is:
$$
-i \hbar \frac{d \Psi^*(x,t)}{dt} = - \frac{\hbar}{2m} \frac{d^2 \Psi^*(x,t)}{dx^2} + V(x,t)\Psi^*(x,t)
$$

#### 📝 Exercise: What is the complex conjugate of
$$
\Psi(x,t) = A e^{(a+bi)(kx - \omega t)}
$$
[Answer](SchrodingerExercise2.ipynb)

### The Hamiltonian
An equation like Schrödinger’s may seem mysterious to us now, but at the time it was derived it seemed very natural. A top-rate physicist like Schrödinger had a rich background in classical physics, and he would have understood that there were many results, from classical physics, that suggested that the equations for radiation and for matter could be combined.  One of the strongest such suggestions came from two principles: “Fermat’s principle of least time” and the “Hamilton-Lagrange principle of least action.” Fermat’s principle says that a ray of light goes from point A to point B in the fastest possible way, and can be used to derive all of classical optics. The Hamilton-Lagrange principle says that a particle goes from point A to point B in a way that makes minimizes its action (the product of its momentum and its change in position) along the path. Schrödinger would have noticed that Planck’s constant had units of action, and then quickly ascertained that laws of quantum mechanics might be governed by an equation similar to the Hamilton-Jacobi equation for the action,

$$\frac{1}{2m}\left(\frac{\partial S}{\partial x}\right)^2 + V(x,t) = \frac{\partial S}{\partial t} $$

or, equivalently,

$$ S \left(-\frac{1}{2m} \frac{\partial^2S}{\partial x^2} + V(x,t) \right) = \frac{\partial S}{\partial t} - \frac{1}{2m}\frac{\partial^2S^2}{\partial x^2}$$

The second term on the right-hand-side of this equation ordinarily integrates to zero (owing to the divergence theorem).

Similarly, as hinted at in the previous set of notes, the Schrödinger equation can be deduced from the (quantum-mechanical) principle of least (really stationary) action. Indeed, the stationary-action principle is more fundamental than the Schrödinger equation, and is the basis for quantum electrodynamics, which is the relativistic theory for matter and electromagnetic radiation that undergirds classical mechanics and classical electromagnetism (in the $\hbar \rightarrow 0$ limit) and the Schrödinger equation (for nonrelativistic matter, without concern for how matter and radiation interact). Feynman and Schwinger won the Nobel Prize for quantum electrodynamics, but its origins go back to Schrödinger's and Dirac's observations about the links between the (least) action principle and the Schrödinger equation.

The Schrödinger equation is usually written in the form:

$$ {\color{blue}\left( -\frac{\hbar^2}{2m} \frac{\partial^2}{\partial x^2}+V(x,t)\right)} \Psi(x,t) = i \hbar \frac{\partial \Psi(x,t)}{\partial t} $$

The term in blue is the [Hamiltonian](https://en.wikipedia.org/wiki/Hamiltonian_(quantum_mechanics)) operator, $\hat{H}(x,t)$; it is the operator for the energy in quantum mechanics. In classical mechanics, the [Hamiltonian](https://en.wikipedia.org/wiki/Hamiltonian_mechanics) is exactly the equation that we started with when we wrote a (classical) expression for the energy of matter, $H(x,t) = \tfrac{p^2}{2m} + V(x,t)$.

Using the Hamiltonian, we can then write the TDSE in a compact form,

$$ {\color{blue}\hat{H}(x,t)}\Psi(x,t) = i \hbar \frac{\partial \Psi(x,t)}{\partial t} $$

As in classical mechanics, the Hamiltonian is the operator for the energy. In quantum mechanics, every observable quantity corresponds to an operator. For example, the kinetic energy operator is

$$ \hat{T} = -\frac{\hbar^2}{2m} \frac{\partial^2}{\partial x^2} $$

Moreover, the values of the properties of any quantum-mechanical operator correspond to its eigenvalues,

$$ (\text{operator}) \Psi = (\text{eigenvalue}) \Psi $$

This suggests that there should be another equation with the form

$$ \hat{H}\Psi = E \Psi $$

This equation is called the [time-independent Schrödinger equation](https://en.wikipedia.org/wiki/Schr%C3%B6dinger_equation#Time-independent_equation) (TISE).

My father, an organic chemist, always says he knew that quantum mechanics would be problematic the first time he saw the Schrödinger equation. He still refers to $\Psi$ as "Satan's pitchfork" and $\hat{H}$ as the "hell operator." Most people, however, call $\hat{H}$ the Hamiltonian and $\Psi$ the wavefunction.

#### 📝 Exercise: Ground-State Energy of the Morse Potential
The Morse potential is often used as an approximate model for the vibrations of diatomic molecules. In convenient units where $\frac{\hslash ^{2}}{2m}=1$, the time-independent Schrödinger equation for a Morse oscillator can be written as:

$$
\left(-\frac{d^{2}}{dx^{2}}+\lambda ^{2}\left(e^{-2x}-2e^{-x}\right)\right)\psi _{n}\left(x\right)=E_{n}\psi _{n}\left(x\right)
$$

The first two eigenfunctions of the Morse oscillator are given by the following expressions (which are not normalized)
$$
\begin{align}
\psi _{0}\left(x\right)&=\exp \left(-\left(\lambda -\tfrac{1}{2}\right)x-\lambda e^{-x}\right)\\
\psi _{1}\left(x\right)&=\exp \left(-\left(\lambda -\tfrac{3}{2}\right)x-\lambda e^{-x}\right)\left(2\lambda -2-2\lambda e^{-x}\right)
\end{align}
$$

**What is the expression for the ground-state energy for the Morse oscillator?**

[Answer](SchrodingerExercise3.md)

### The Time-Independent Schrödinger Equation (TISE)
In much of chemistry, the Hamiltonian operator is not time-dependent.  For examples, the electrons in a molecule feel a fixed (time-independent) attraction to the nuclei in the molecule and, to a first approximation, the (very small) motion of the nuclei can be neglected.  Similarly, in the absence of a time-varying external field, a molecule as a whole (containing both electrons and nuclei) does not feel a time-varying potential.

To derive the time-independent Schrödinger equation, we insert the time-independent Hamiltonian,

$$\hat{H}(x) = -\frac{\hbar^2}{2m} \frac{\partial^2}{\partial x^2} +V(x)$$

into the time-dependent Schrödinger equation,

$$ \hat{H}(x)\Psi(x,t) = i \hbar \frac{\partial \Psi(x,t)}{\partial t} $$

and then rearrange the equation into the form

$$ \left( \hat{H}(x) - i\hbar \frac{\partial}{\partial t} \right) \Psi(x,t) = 0 $$

This is a differential equation that can be solved by the general technique of [separation of variables](https://en.wikipedia.org/wiki/Separation_of_variables). Specifically,

> [**Separation of Variables**](https://en.wikipedia.org/wiki/Separation_of_variables#Partial_differential_equations) Given an operator that is a sum of two operators that depend on different (sets of) variables,

$$ \hat{A}(x,y) = \hat{A}_x(x) + \hat{A}_y(y) $$

The solution to the eigenvalue equation

$$ \hat{A}(x,y) \Psi(x,y) = a \Psi(x,y) $$

is

$$\Psi(x,y) = \Psi_x(x) \cdot \Psi_y(y) $$

$$ a = a_x + a_y $$

where

$$\hat{A}_x(x) \Psi_x(x) = a_x \Psi_x(x) $$

$$\hat{A}_y(y) \Psi_y(y) = a_y \Psi_y(y) $$

This tells us that for a time-independent Hamiltonian, the wavefunction has the form $\Psi(x,t) = \psi(x) \phi(t)$, where

$$\hat{H}(x) \psi(x) = E \psi(x) $$

$$- i\hbar \frac{\partial}{\partial t} \phi(t) = -E\phi(t) $$

In the first equation, we have chosen the eigenvalue of the TDSE to be the energy since we know that the Hamiltonian is the quantum-mechanical operator for the energy. The second equation is a separable differential equation, which can be solved as follows:

$$
\begin{align}
-i\hbar \frac{\partial}{\partial t} \phi(t) &= -E\phi(t) \\
\frac{1}{\phi} d\phi &= -\frac{i}{\hbar}E dt \\
\int \frac{1}{\phi} d\phi = \int \frac{-i}{\hbar}E dt \\
\ln \phi(t) &= \frac{-iEt}{\hbar} + \text{constant of integration}\\
\phi(t)  &\propto e^{\frac{-iEt}{\hbar}}
\end{align}
$$

Therefore,

$$ \Psi(x,t) = \psi(x) \phi(t) \propto \psi(x) e^{\frac{-iEt}{\hbar}} $$

The constant of proportionality is not important here, because the time-dependent and time-independent Schrödinger equations are satisfied even if $\Psi(x)$ and $\phi(t)$ are multiplied by constants.

## Summary
When solving the time-independent Schrödinger equation, we need to find all the possible values of E that solve this equation. There are almost always very many different values of E that work and, for this reason, we usually label the solutions to the equation, which are the eigenfunctions (eigenvectors) and eigenvalues of the Hamiltonian operator, accordingly:

$$ \hat{H}(x)\psi_k(x) = E_k \psi_k(x) $$

This course is largely about the “art” of solving the TISE

Sometimes in chemistry we cannot use the time-independent Schrödinger equation, and need to use the time-dependent Schrödinger equation instead. This is particular important in spectroscopy, where we want to understand how a molecule responds to a time-dependent external electromagnetic field.

## 🪞 Self-Reflection
The Hamiltonian operator enters the Schrödinger equation in a special way, and in this sense is a "more important" operator than other quantum-mechanical operators. Why do you think this is the case?

## ❓ Knowledge Tests
- Questions about the Schrodinger Equation. [GitHub Classroom Link](https://classroom.github.com/a/yXuh6NJt)

## 🤔 Thought-Provoking Questions
- Once upon a time, one of the professors at McMaster was asked "why is the quantum-mechanical momentum operator associated with the derivative" and, unable to answer the student, sent the student to me for clarification. Can you justify why the quantum mechanical momentum operator is $\hat{p} = -i \hbar \nabla$?
- Once upon a time, a person interviewing for a job as a quantum chemist was asked how it was possible that the kinetic energy of a quantum system was always positive, given that the quantum-mechanical operator for the kinetic energy was "the negative of something squared", namely, $\hat{T} = \tfrac{-1}{2m}\nabla^2$. How would you answer this question?
- What are the eigenfunctions of the momentum operator?
- What are the eigenfunctions of the kinetic-energy operator?
- Is every eigenfunction of the momentum operator also an eigenfunction of the kinetic-energy operator? Is the [converse](https://en.wikipedia.org/wiki/Converse_(logic)) also true?
- Suppose that one has an electron that is tethered to the origin by a spring, so that the force the electrons feels towards the origin, $x=0$, increases proportionally to its distance, $F = -kx$. What would the Hamiltonian be?
- What are the eigenfunctions of the kinetic-energy operator?
- What are the eigenfunctions of the momentum operator?
- Show, mathematically, that the eigenvectors and eigenvalues of a separable operator, $\hat{A}(x,y) = \hat{A}(x) + \hat{A}(y)$ are given by the above expressions.

## 🔁 Recapitulation
- Write the quantum-mechanical operator for the energy, kinetic-energy, potential energy, and momentum.
- Write the time-independent and time-dependent Schrödinger equations.
- What is the form of the time-dependent wavefunction that solves the TDSE when the Hamiltonian is time-independent.

## 🔮 Next Up...
- Study a particle confined to a box with infinite sides.
- Explore the postulates of quantum mechanics.

## 📚 References
My favorite sources for this material are:
- R. Eisberg and R. Resnick, Quantum Physics of Atoms, Molecules, Solids, Nuclei, and Particles (Wiley, New York, 1974)
- R. Dumont, [An Emergent Reality, Part 2: Quantum Mechanics](https://github.com/PaulWAyers/IntroQChem/blob/main/documents/DumontBook.pdf?raw=true) (Chapters 1 and 2).
- Also see my (pdf) class [notes](https://github.com/PaulWAyers/IntroQChem/blob/main/documents/IntroQM.pdf?raw=true).
- [Davit Potoyan's](https://www.chem.iastate.edu/people/davit-potoyan) Jupyter-book course "derives" the Schrodinger equation in a somewhat different way. [Chapter 3](https://dpotoyan.github.io/Chem324/Lec3-0.html) is especially relevant here.

There are also some excellent wikipedia articles:
- [Plane Wave](https://en.wikipedia.org/wiki/Sinusoidal_plane_wave)
- [Wavelength-Frequency Relation](https://en.wikipedia.org/wiki/Wavelength)
- Schrödinger Equation
  - [Schrödinger Equation](http://en.wikipedia.org/wiki/Schrodinger_equation)
  - [Theoretical Justification of S.E.](http://en.wikipedia.org/wiki/Theoretical_and_experimental_justification_for_the_Schr%C3%B6dinger_equation).
  - [Momentum Operator](http://en.wikipedia.org/wiki/Momentum_operator)