```julia
using Pkg
if isfile("../Project.toml") && isfile("../Manifest.toml")
    Pkg.activate("..");
end

using Random
using Distributions
using Plots
using StatsPlots
using prml
using prml: pdf

Random.seed!(1234);
rng = MersenneTwister(1234);
gr();
```

```julia
function create_toy_data(func, sample_size, std, domain=[0., 1.])
    x = collect(range(domain[1], stop=domain[2], length=sample_size));
    shuffle!(rng, x);
    noise = rand(Normal(0.0, std), sample_size);
    return x, func(x) + noise
end

function sinusoidal(x)
    return sin.(2 * pi * x)
end
```

## Polynomial, Gaussian, Sigmoid features

```julia
x = collect(range(-1.0, stop=1.0, length=100));
means = collect(range(-1.0, stop=1.0, length=12));
feature_poly = PolynomialFeature(11);
feature_gauss = GaussianFeature(means, 0.1);
feature_sigmoid = SigmoidalFeature(means, 10.0);

X_polynomial = transform(feature_poly, x);
X_gaussian = transform(feature_gauss, x);
X_sigmoidal = transform(feature_sigmoid, x);

# polynomial
p1 = plot();
for j in 1:12
    p1 = plot!(x, X_polynomial[:, j]);
end
# gaussian
p2 = plot();
for j in 1:12
    p2 = plot!(x, X_gaussian[:, j]);
end
# sigmoidal
p3 = plot();
for j in 1:12
    p3 = plot!(x, X_sigmoidal[:, j]);
end
plot(p1, p2, legend=nothing)
# plot(p1, p2, p3, legend=nothing)
```

## Ridge regression

$$
\boldsymbol{w} = (\boldsymbol{\Phi}^{T} \boldsymbol{\Phi} + \lambda I)^{-1} \boldsymbol{\Phi}^{T} \boldsymbol{t}
$$

```julia
x_train, y_train = create_toy_data(sinusoidal, 10, 0.4);
x_test = collect(range(0, stop=1.0, length=100));
y_test = sinusoidal(x_test);
```

```julia
means = collect(range(0, stop=1.0, length=7));
feature = GaussianFeature(means, 0.1);

X_train = transform(feature, x_train);
X_test = transform(feature, x_test);
model = LinearRegressor([0], 0);
fitting(model, X_train, y_train);

y, var = predict(model, X_test, true);
y_std = fill(var, size(y)[1]);

plot(x_test, y_test, label="sin(2π x)", color="steelblue", lw=2);
plot!(x_train, y_train, label="training data", seriestype=:scatter);
plot!(x_test, y, ribbon=(y_std, y_std), fillalpha=0.5, label="std")
```

```julia
model = RidgeRegressor([0.0], 1e-2);
fitting(model, X_train, y_train);
y = predict(model, X_test);

plot(x_train, y_train, label="training data", seriestype=:scatter);
plot!(x_test, y, label="predict", lw=2)
plot!(x_test, y_test, label="sin(2π x)", lw=2)
```

## Bias-Variance decomposition

```julia
means = collect(range(0, stop=1.0, length=24));
feature = GaussianFeature(means, 0.1);

plots = [];
for a in [1e2, 1.0, 1e-9]
    y_list = [];
    p = plot();
    for i in 1:100
        x_train, y_train = create_toy_data(sinusoidal, 25, 0.25);
        x_test = collect(range(0, stop=1.0, length=100));
        X_train = transform(feature, x_train);
        X_test = transform(feature, x_test);
        model = BayesianRegressor(a, 1.0, 24);
        fitting(model, X_train, y_train);
        y = predict(model, X_test);
        push!(y_list, y);
        if i < 20
            p = plot!(x_test, y, color="orange");
        end
    end

    push!(plots, p);
    p = plot();
    p = plot!(x_test, y_test, color="steelblue", lw=2);
    p = plot!(x_test, mean(y_list), color="orange", lw=2);
    push!(plots, p);
end

plot(plots[1], plots[2], plots[3], plots[4], plots[5], plots[6], layout=(3, 2), ylim=(-1.5, 1.5), legend=nothing)
```

## Bayesian learning

```julia
function linear(x)
    return -0.3 .+ 0.5 .* x
end

x_train, y_train = create_toy_data(linear, 20, 0.1, [-1., 1.]);
x = collect(range(-1., stop=1., length=100));
y = collect(range(-1., stop=1., length=100));
#grids = [[i, j] for i in x, j in y];

model = BayesianRegressor(1.0, 100.0, 2);
feature = PolynomialFeature(1);
X_train = transform(feature, x_train);
X = transform(feature, x);

dist = MvNormal(model._w_mean, inv(model._w_precision));
vals = [Distributions.pdf(dist, [i, j]) for j in y, i in x];

plots = [];
p = contour(x, y, vals, colorbar=false);
p = plot!([-0.3], [0.5], seriestype=:scatter, marker=:dot, markersize=:4, markercolor="blue");
push!(plots, p);

Y_random = predictSampling(model, X, 6);
p = plot();
for i in 1:6
    p = plot!(x, Y_random[i, :], c="orange");
end
push!(plots, p);

for (index, values) in enumerate([[1, 1], [2, 2], [3, 3], [4, 20]])
    first = values[1];
    last = values[2];
    fitting(model, X_train[first:last, :], y_train[first:last]);
    dist = MvNormal(model._w_mean, inv(model._w_precision));
    vals = [Distributions.pdf(dist, [i, j]) for j in y, i in x];
    p = contour(x, y, vals, colorbar=false);
    p = plot!([-0.3], [0.5], seriestype=:scatter, marker=:dot, markersize=:4, markercolor="blue");
    push!(plots, p);

    Y_random = predictSampling(model, X, 6);
    p = plot();
    for i in 1:6
        p = plot!(x, Y_random[i, :], c="orange");
    end
    p = plot!(x_train[1:last], y_train[1:last], c="steelblue", seriestype=:scatter);
    push!(plots, p);
end

plot(plots[1], plots[2], plots[3], plots[4], plots[5], plots[6], plots[7], plots[8], plots[9], plots[10], layout=(5, 2), size=(600, 1250), legend=nothing)
```

## Bayesian linear regression

### Regression

The data is generated from the distribution

$$
\begin{align} t = y(\boldsymbol{x}, \boldsymbol{w}) + \epsilon \end{align}.
$$

Or

$$
\begin{align} p(t \mid \boldsymbol{w}, \beta) = \mathcal{N}(t \mid y(\boldsymbol{x}, \boldsymbol{w}), \beta^{-1}) \end{align}
$$

And the training data is denoted as $\boldsymbol{X} = [\boldsymbol{x}_1, \cdots \boldsymbol{x}_N]$, and $\boldsymbol{t} = [t_1, \cdots t_N]$. The likelihood for the traning data is then given by

$$
\begin{align} p(\boldsymbol{t} \mid \boldsymbol{X}, \boldsymbol{w}, \beta) = \prod \mathcal{N}(t_n \mid \boldsymbol{w}^{\text{T}} \boldsymbol{\phi}(\boldsymbol{x}_n), \beta^{-1}) \end{align}
$$

Now we assume the prior distribution for $\boldsymbol{w}$ is $\mathcal{N}(\boldsymbol{w} \mid \boldsymbol{m}_0, S_0)$. Then the Bayes's theorem gives the posterior distribution for $\boldsymbol{w}$ as follows,

$$
\begin{align} p(\boldsymbol{w} \mid \boldsymbol{t}_N) &= \mathcal{N}(\boldsymbol{m}_N, S_N) \\ \boldsymbol{m}_N &= S_N(S_0^{-1} \boldsymbol{m}_0 + \beta \Phi_N^{\text{T}}\boldsymbol{t}_N) \\ S_N^{-1} &= S_0^{-1} + \beta \Phi_N^{\text{T}}\Phi_N \end{align}
$$

where

$$
\Phi_{N}^{\text{T}} = \begin{bmatrix} \boldsymbol{\phi}_{1} & \boldsymbol{\phi}_{2} & \cdots & \boldsymbol{\phi}_{N} \end{bmatrix}, \quad
\Phi_N = \begin{bmatrix} \boldsymbol{\phi}_{1}^{\text{T}} \\ \boldsymbol{\phi}_{2}^{\text{T}} \\ \vdots \\ \boldsymbol{\phi}_{N}^{\text{T}} \end{bmatrix}
$$

### Predictive distribution

$$
p(t \mid \boldsymbol{x}, \alpha, \beta) = \mathcal{N} \left(t \mid\boldsymbol{m}_{N}^{\text{T}}\phi(\boldsymbol{x}), \sigma^2(\boldsymbol{x}) \right)
$$

where

$$
\sigma^2(\boldsymbol{x}) = \dfrac{1}{\beta} + \phi(\boldsymbol{x})^{\text{T}} S_N \phi(\boldsymbol{x})
$$

As you can see, the variance is dependent on $\boldsymbol{x}$.

```julia
x_train, y_train = create_toy_data(sinusoidal, 25, 0.25);
x_test = collect(range(0, stop=1.0, length=100));
y_test = sinusoidal(x_test);

means = collect(range(0, stop=1.0, length=9));
feature = GaussianFeature(means, 0.1);
X_train = transform(feature, x_train);
X_test = transform(feature, x_test);

model = BayesianRegressor(1e-3, 2.0, 9);
plots = [];

for (index, ranges) in enumerate([[1, 1], [2, 2], [3, 4], [5, 9], [10, 15], [16, 25]])
    first, last = ranges[1], ranges[2];
    fitting(model, X_train[first:last, :], y_train[first:last]);
    y, y_std = predict(model, X_test, true);
    p = plot(x_train[1:last], y_train[1:last], seriestype=:scatter);
    p = plot!(x_test, y_test, lw=2, color="steelblue");
    p = plot!(x_test, y, ribbon=(y_std, y_std), fillalpha=0.5, color="orange", lw=2);
    push!(plots, p);
end

l = @layout [a b; c d; e f];

plot(plots[1], plots[2], plots[3], plots[4], plots[5], plots[6], xlim=(0, 1), ylim=(-2, 2), layout=l, legend=nothing, size=(600, 750))
```

## Evidence approximiation

$$\begin{align}
\ln p(\boldsymbol{t} \mid \alpha, \beta) &= \dfrac{M}{2} \ln \alpha + \dfrac{N}{2} \ln \beta - E(\boldsymbol{w}) - \dfrac{1}{2} \ln \det A - \dfrac{N}{2} \ln (2 \pi) \\
A &= \alpha I + \beta \Phi^{\text{T}}\Phi \\
E(\boldsymbol{w}) &= \dfrac{\beta}{2} \| \boldsymbol{t} - \Phi \boldsymbol{w} \| + \dfrac{\alpha}{2} \boldsymbol{w}^{\text{T}} \boldsymbol{w}
\end{align}$$

Minimization of $\ln p(\boldsymbol{t} \mid \alpha, \beta)$ is achieved as follows.

1. $\alpha \leftarrow \alpha_0$
2. $\beta \leftarrow \beta_0$
3. **while**  $(\alpha, \beta)$ **converges do**:
4.  $\boldsymbol{m}_N \leftarrow \beta(\alpha I + \beta \Phi^{\text{T}}\Phi)^{-1}\Phi^{\text{T}}\boldsymbol{t}$
5. $\boldsymbol{\lambda} \leftarrow \mathrm{eig}(\beta \Phi^{\text{T}}\Phi)$
6. $\gamma \leftarrow \sum_i \lambda_i / (\alpha + \lambda_i)$
7. $\alpha \leftarrow \gamma / |\boldsymbol{m}_N|$
8. $1 / \beta \leftarrow \sum_{n=1}^{N}(\boldsymbol{m}_N^{\text{T}}\boldsymbol{\phi}(\boldsymbol{x}_n) - t_n)^2 / (N - \gamma)$
9. **end while**
10. **return**  $(\alpha, \beta)$

```julia
function cubic(x)
    return x .* (x .- 5.0) .* (x .+ 5.0)
end

x_train, y_train = create_toy_data(cubic, 30, 10, [-5.0, 5.0]);
x_test = collect(range(-5.0, stop=5.0, length=100));
evidences = [];
models = [];

for i in 1:6
    feature = PolynomialFeature(i);
    X_train = transform(feature, x_train);
    model = EmpiricalBayesianRegressor(100.0, 100.0, i+1);
    fitting(model, X_train, y_train, 100);
    push!(evidences, log_evidence(model, X_train, y_train));
    push!(models, model);
end

plot(evidences, xlabel="degree", ylabel="log evidence", title="Model Evidence", legend=nothing, lw=2)
```

```julia
degree = findall(isequal(minimum(evidences)), evidences);
regressor = models[degree[1]];
feature = PolynomialFeature(degree[1]);

X_test = transform(feature, x_test);
y, y_std = predict(regressor, X_test, true);
plot(x_test, cubic(x_test), label="x(x-5)(x+5)", lw=2)
plot!(x_test, y, ribbon=(y_std, y_std), label="prediction", fillalpha=0.5, color="orange", lw=2)
plot!(x_train, y_train, seriestype=:scatter, label="training data")
```


Table of Contents

1  Objectives

2  What & Why of Linear Algebra

3  Scalars, Vectors, and Tensors! Oh My!

3.1  Scalars

3.2  Vectors

3.2.1  Code for Vectors

3.2.2  Math with Vectors

3.2.2.1  Vector Addition

3.2.2.2  Vector Multiplication

3.3  Matrices and Tensors

3.3.1  Code for Matrices and Tensors

3.3.2  Math with Tensors

3.3.2.1  Addition

3.3.2.2  Dot-Product

4  More with Matrices

4.1  Identity Matrices

4.1.1  Identity Matrices in NumPy

4.2  Inverse Matrices

5  Solving a System of Linear Equations

5.1  Representing the System with Matrices

5.1.1  Coding It with NumPy

5.1.2  Solve It Faster with NumPy's linalg.solve()

6  Solving for the Line of Best Fit: Linear Regression

6.1  Linear Algebra Solves the Best-Fit Line Problem

```python
import numpy as np
from sklearn.linear_model import LinearRegression
```

# Objectives

- Use `numpy` to construct and manipulate scalars, vectors, and matrices
- Use `numpy` to construct and manipulate identity matrices and inverse matrices
- Use `numpy` to solve systems of equations
- Describe the matrix manipulations required to solve the best-fit line problem

# What & Why of Linear Algebra

Matrices are a fundamental aspect of data science models and problems, including image processing, deep learning, NLP, and PCA. You will encounter matrices *many* times in your career as a data scientist. Matrices are a fundamental tool in **linear algebra**.

-----------

- Study of "vector spaces"; relationship of **linear** relationships
- Uses vectors, matrices, and tensors
- Mapping & dimensionality (PCA)
- Used in lots of ML applications

We'll try to put abstract ideas into the formalism of linear algebra, such as:
- data values
- images/pixels
- language (NLP)

# Scalars, Vectors, and Tensors! Oh My!

> You can think of these values being built up to higher dimensions

## Scalars

A _scalar_ has simply a single value. Any real number can be the value of a scalar.

```python
# Scalar
s = np.arange(1)
display(s)
```

## Vectors

A _vector_ can be specified by with just _two_ parameters: magnitude and direction. To remind of this direction, a vector is typically denoted with an arrow above the variable representing it: $\vec{v}$ .

In a Cartesian coordinate system, a vector $\vec{v}$ will often be specified by the components defined by the coordinate system.

In this way a vector can be embedded in a higher-dimensional space, which allows us to speak of vectors of any dimension (even though, as directed line segments, all vectors are, strictly speaking, one-dimensional).

> Svjo, CC BY-SA 4.0, via Wikimedia Commons

For more on vectors, see this helpful [video](https://www.youtube.com/watch?v=fNk_zzaMoSs&list=PLZHQObOWTQDPD3MizzM2xVFitgF8hE_ab).

### Code for Vectors

> Note we can think of vectors as a one-dimensional object residing in multidimensional space.

```python
# Vector
v = np.arange(4)
display(v)
print('Shape:', np.shape(v))
```

```python
# Vector
v = np.array([1,0,2,1,0,1,0,1,1])
display(v)
print('Shape:', np.shape(v))
```

```python
# Other ways to define vector
x = np.linspace(-np.pi, np.pi, 10)
display(x)
print('Shape:', np.shape(v))
```

These vectors above are usually called _row vectors_.

We can also specify _column vectors_.

```python
# Column vector from scratch
col_vector = np.array([[1],[0],[2],[1],[0],[1],[0],[1],[1]])
display(col_vector)
print('Shape:', np.shape(col_vector))
```

```python
# Reformatting or "reshaping" a row vector to a column vector
display(v)
print('Shape:', np.shape(v))

print('')
print('='*64)

col_vector = v.reshape(-1,1) # This is a common way for row -> column
display(col_vector)
print('Shape:', np.shape(col_vector))
```

### Math with Vectors

#### Vector Addition

Vector addition is simple: Just add the corresponding components together:

$[8, 14] + [7, 6] = [15, 20]$

Base Python is not particularly good for non-scalar arithmetic. Make a general practice of turning to `numpy` for mathematical operations.

```python
# Let's try this again, but this time we'll use NumPy arrays:

vec_1 = np.array([8, 14])
vec_2 = np.array([7, 6])
vec_1 + vec_2
```

#### Vector Multiplication

In fact there are multiple ways of understanding the notion of vector multiplication. All are potentially useful, but the one that we'll likely be of most use is the *dot-product*, which is defined as follows:

$$
\begin{equation}
\begin{bmatrix}
a \\
b
\end{bmatrix}
.
\begin{bmatrix}
c \\
d
\end{bmatrix}
=
ac + bd
\end{equation}
$$

The dot-product is the sum of the pariwise products of the vectors' entries.

```python
# Let's check out the different attributes and methods available to
# a NumPy array.

# vec_1.

# There are many options. Notice that one of these options is 'dot'.
# This is our dot-product! So let's use the .dot() method to calculate
# the dot-product of our two vectors:

vec_1.dot(vec_2)
```

```python
# We can also use '@'

vec_1 @ vec_2
```

## Matrices and Tensors

For higher dimensions we can use **matrices** to express ourselves. Suppose we had a two-variable system:

$$
\begin{align}
a_{1,1}x_1 + a_{1,2}x_2 = c_1 \\
a_{2,1}x_1 + a_{2,2}x_2 = c_2
\end{align}
$$

Using matrices, we can write this as:

$$
\begin{equation}
\begin{bmatrix}
a_{1,1} & a_{1,2} \\
a_{2,1} & a_{2,2}
\end{bmatrix}
\begin{bmatrix}
x_1 \\
x_2
\end{bmatrix} =
\begin{bmatrix}
c_1 \\
c_2
\end{bmatrix}
\end{equation}
$$

or

$A\vec{x} = \vec{c}$,

where:

- $\vec{x}$ is the _vector_ $(x_1, x_2)$;
- $\vec{c}$ is the _vector_ $(c_1, c_2)$; and
- $A$ is the _matrix_ of coefficients that describe our system:

$\begin{equation} A =
\begin{bmatrix}
a_{1,1} & a_{1,2} \\
a_{2,1} & a_{2,2}
\end{bmatrix}
\end{equation}$

------

Technically, all of what we've talked about are **tensors**, just different ranks:

- scalar == 0th rank tensor
- vector == 1st rank tensor
    + vectors are made up from a "list" of scalars
- matrix == 2nd rank tensor
    + matrices are made up from a "list" of vectors
- 3D matrix == 3rd rank tensor
    + 3D matrices are made up from a "list" of (2D) matrices
- and so on...

> However, people are referring to the higher ranked tensors (2nd and above) when they say "tensor".

### Code for Matrices and Tensors

```python
# Matrix
M = np.arange(4 * 2).reshape((4, 2))
display(M)
```

```python
# 3D Tensor
T_3d = np.arange(4 * 2 * 3).reshape((4, 2, 3))
display(T_3d)
```

### Math with Tensors

#### Addition

The addition of matrices is straightforward: Just add corresponding elements. In order to add two matrices $A$ and $B$, they must have the same number of rows and columns:

$\begin{bmatrix}
a_{11} & a_{12} \\
a_{21} & a_{22}
\end{bmatrix}
+
\begin{bmatrix}
b_{11} & b_{12} \\
b_{21} & b_{22}
\end{bmatrix}
=
\begin{bmatrix}
a_{11} + b_{11} & a_{12} + b_{12} \\
a_{21} + b_{21} & a_{22} + b_{22}
\end{bmatrix}
$

```python
A = np.array([[1, 2], [3, 4]])
B = np.array([[4, 3], [2, 1]])
display(A)
display(B)
```

```python
A + B
```

#### Dot-Product

> Just as there were different notions of "multiplication" for vectors, so too there are different notions of multiplication for matrices.

Very often when people talk about multiplying matrices they'll mean the dot-product:

$$
\begin{equation}
\begin{bmatrix}
a_{1,1} & a_{1,2} \\
a_{2,1} & a_{2,2}
\end{bmatrix}
\cdot
\begin{bmatrix}
b_{1,1} & b_{1,2} \\
b_{2,1} & b_{2,2}
\end{bmatrix}
=
\begin{bmatrix}
a_{1,1}\times b_{1,1} + a_{1,2}\times b_{2,1} & a_{1,1}\times b_{1,2} + a_{1,2}\times b_{2,2} \\
a_{2,1}\times b_{1,1} + a_{2,2}\times b_{2,1} & a_{2,1}\times b_{1,2} + a_{2,2}\times b_{2,2}
\end{bmatrix}
\end{equation}
$$

Take the entries in each *row* of the left matrix and multiply them, respectively, by the entries in each *column* of the right matrix, and then add them up. This is the product we calculated above with our two vectors!

We can multiply in NumPy in the same way of doing a dot product with vectors.

```python
A = np.array([[1,2,3],[4,5,6]])
B = np.array([[11,22],[33,44],[55,66]])

# Different ways to do the same dot product
AB = np.dot(A,B)
AB = A.dot(B)
AB = A @ B
```

Observe also that in order to be able to perform the dot product on two matrices A and B, **the number of columns of A must equal the number of rows of B**.

Also, **the number of rows of the product matrix will equal the number of rows of A, and the number of columns of the product matrix will equal the number of columns of B**.

> **A note about vectors and matrices**
>
> Matrix dot-multiplication is NOT commutative! In general, $AB \neq BA$.
>
> Strictly speaking, this is true for vectors as well. Above, we multiplied the *row*-vector $(a, b)$ by the *column*-vector $(c, d)$. A row-vector is simply a matrix with only one row; a column-vector is simply a matrix with only one column.
>
> What would be the result of multiplying the column-vector $(c, d)$ on the left by the row-vector $(a, b)$ on the right?
>
> $$\begin{equation}
    \begin{bmatrix}
        c \\
        d
    \end{bmatrix}
    \space
    \begin{bmatrix}
        a & b
    \end{bmatrix}
    =
    \begin{bmatrix}
        ca & cb \\
        da & db
    \end{bmatrix}
  \end{equation}$$

##### Exercise

Illustrate this difference between $\begin{bmatrix} a & b \end{bmatrix} \begin{bmatrix} c \\ d \end{bmatrix}$ and $\begin{bmatrix} c \\ d \end{bmatrix} \begin{bmatrix} a & b \end{bmatrix}$ in Python for $a=2, b=3, c=4$, and $d=5$.

```python
# Your code here

```

    Solution

```python
    ab = np.array([[2, 3]])
    cd = np.array([[4],[5]])

    display(ab @ cd)
    display(cd @ ab)

```

# More with Matrices

In order to solve an equation like $A\vec{x} = \vec{c}$ for $\vec{x}$, we can't very well divide $\vec{c}$ by $A$! But there is a notion of matrix _inversion_ that is relevant here, which is analogous to multiplicative inversion. If we have an equation like $2x = 10$, we can simply multiply both sides by the multiplicative inverse of the coefficient of $x$, viz. $2^{-1}$. And here the point, of course, is that $2^{-1} \times 2 = 1$.

We'll see there is an equivalent _inverse_ for matrices. But first, we should discuss the identity matrix.

## Identity Matrices

> The **identity matrix** $I$ is the matrix containing 1's along the main diagonal (upper-left to lower-right) and 0's everywhere else:
>
> $$\begin{align}
    I_3 &= \begin{bmatrix} 1 & 0 & 0 \\ 0 & 1 & 0 \\ 0 & 0 & 1 \end{bmatrix} \\
    \\
    I_5 &= \begin{bmatrix} 1 & 0 & 0 & 0 & 0 & 0 \\
                           0 & 1 & 0 & 0 & 0 & 0 \\
                           0 & 0 & 1 & 0 & 0 & 0 \\
                           0 & 0 & 0 & 1 & 0 & 0 \\
                           0 & 0 & 0 & 0 & 1 & 0 \\
                           0 & 0 & 0 & 0 & 0 & 1 \\
            \end{bmatrix}
\end{align}$$

The identity matrix is a special matrix since any (compatible) matrix $A$ multiplied by it is the matrix itself: $AI = A$ and $IA = A$

###  Identity Matrices in NumPy

```python
I5 = np.eye(5)
print(I5)
```

```python
A = 42*np.ones(25).reshape(5,5)
A
```

```python
print(I5 @ A)
print()
print(A @ I5)
print()
is_equal = (I5 @ A) == (A @ I5)
print('Both are the same:')
print(is_equal)
```

## Inverse Matrices

In the higher-dimensional case, what we can do is to left-multiply both sides by the _inverse matrix_ of A, denoted $A^{-1}$, and here the point is that the dot-product $A^{-1}A = I$, where $I$ is the identity matrix.

Thus when we have a matrix equation $A\vec{x} = \vec{c}$, we can calculate the solution by multiplying both sides by $A^{-1}$:

$$\begin{align}
    A\vec{x}       &= \vec{c}\\
    A^{-1}A\vec{x} &= A^{-1}\vec{c}\\
    I \vec{x}      &= A^{-1}\vec{c}\\
    \vec{x}        &= A^{-1}\vec{c}\\
\end{align}$$

```python
# Your code here
A = np.array([
    [ 1,-2, 3],
    [ 2,-5,10],
    [ 0, 0, 1]
])

```

```python
np.linalg.inv(A)
```

##### Exercise

You can produce the inverse of a matrix in `numpy` by calling `np.linalg.inv()`.

Using `numpy` arrays, find the inverse $A^{-1}$ of the matrix below:

$$A = \begin{bmatrix} 1 & -2 & 3 \\ 2 & -5 & 10 \\ 0 & 0 & 1 \end{bmatrix}$$

> Confirm this is the inverse by multiplying the matrices

```python
# Your code here

```

# Solving a System of Linear Equations

Solving a system of equations can take a lot of work

$$ \begin{align}
 1 - 2y + 3z &= 9 \\
 2x - 5y + 10z &= 4 \\
  6z &= 0
\end{align}$$

## Representing the System with Matrices

But we can make it easier by writing it in matrix form

$$
\begin{pmatrix}
    1 & -2 & 3 \\
    2 & -5 & 10 \\
    0 & 0 & 6
\end{pmatrix}
\cdot
\begin{pmatrix}
    x \\
    y \\
    z
\end{pmatrix}
=
\begin{pmatrix}
    9 \\
    4 \\
    0
\end{pmatrix}
$$

We can think of this in the abstract:
$$ A \cdot X = B $$
$$ A^{-1} \cdot A \cdot X = A^{-1} \cdot B $$
$$ I \cdot X = A^{-1} \cdot B $$
$$ X = A^{-1} \cdot B $$

### Coding It with NumPy

Let's try solving the system with matrices:

$$ \begin{align}
 1 - 2y + 3z &= 9 \\
 2x - 5y + 10z &= 4 \\
  6z &= 0
\end{align}$$

First define the system's matrices:

$$
\begin{pmatrix}
    1 & -2 & 3 \\
    2 & -5 & 10 \\
    0 & 0 & 6
\end{pmatrix}
\cdot
\begin{pmatrix}
    x \\
    y \\
    z
\end{pmatrix}
=
\begin{pmatrix}
    9 \\
    4 \\
    0
\end{pmatrix}
$$

to

$$ A \cdot \vec{X} = B $$

```python
A = np.array([
    [1, -2,  3],
    [2, -5, 10],
    [0,  0,  6]
])
```

```python
B = np.array([9,4,0]).reshape(3,1)
```

```python
print('A:')
print(A)
print()
print('B:')
print(B)
```

Find the inverse

```python
A_inv = np.linalg.inv(A)
print(A_inv)
```

Getting the solution

```python
solution = A_inv @ B
print(solution)
```

### Solve It Faster with NumPy's `linalg.solve()`

NumPy's ```linalg``` module has a ```.solve()``` method that you can use to solve a system of linear equations!

In particular, it will solve for the vector $\vec{x}$ in the equation $A\vec{x} = b$. You should know that, "under the hood", the ```.solve()``` method does NOT compute the inverse matrix $A^{-1}$. This is largely because of the enormous expense of directly computing a matrix inverse, which takes $\mathcal{O}(n^3)$ time.

Check out [this discussion](https://stackoverflow.com/questions/31256252/why-does-numpy-linalg-solve-offer-more-precise-matrix-inversions-than-numpy-li) on stackoverflow for more on the differences between using `.solve()` and `.inv()`.

And check out the documentation for ```.solve()``` [here](https://docs.scipy.org/doc/numpy/reference/generated/numpy.linalg.solve.html).

```python
# Let's use the .solve() method to solve this system of equations

A = np.array([
    [1, -2,  3],
    [2, -5, 10],
    [0,  0,  6]
])

B = np.array([9,4,0]).reshape(3,1)
```

```python
np.linalg.solve(A, B)
```

Again, we could just solve our matrix equation by calculating the inverse of our matrix $X$ and then multiplying by $y$:

```python
np.linalg.inv(A).dot(B)
```

But the time difference is striking:

```python
%timeit np.linalg.inv(A).dot(B)
```

```python
%timeit np.linalg.solve(A, B)
```

Even for a (tiny!) 5x5 matrix, the cost of computing the inverse directly is evident.

# Solving for the Line of Best Fit: Linear Regression

Consider a typical dataset and the associated multiple linear regression problem. We have many observations (rows), each of which consists of a set of values both for the predictors (columns, i.e. the independent variables) and for the target (the dependent variable).

We can think of the values of the independent variables as our matrix $A$ of coefficients and of the values of the dependent variable as our output vector $\vec{c}$.

The task here is, in effect, to solve for $\vec{\beta}$, where we have that $A\vec{\beta} = \vec{c}$, except in general we'll have more rows than columns. This is why we won't in general be computing matrix inverses. (They're computationally expensive, anyway.) This is also why we have a problem requiring not a direct solution but rather an optimization--in our case, a best-fit line.

Using $z$ for our independent variables and $y$ for our dependent variable, we have:

\begin{equation}
\beta_1\begin{bmatrix}
z_{1,1} \\
. \\
. \\
. \\
z_{m,1}
\end{bmatrix} +
... + \beta_n\begin{bmatrix}
z_{1,n} \\
. \\
. \\
. \\
z_{m,n}
\end{bmatrix} \approx \begin{bmatrix}
y_1 \\
.  \\
.  \\
.  \\
y_m
\end{bmatrix}
\end{equation}

## Linear Algebra Solves the Best-Fit Line Problem

If we have a matrix of predictors $X$ and a target column $y$, we can express $\hat{y}$, the best-fit line, as  follows:

$\large\hat{y} = (X^TX)^{-1}X^Ty$.

$(X^TX)^{-1}X^T$ is sometimes called the *pseudo-inverse* of $X$. We'll have more to say about this in a future lesson when we talk about the singular value decomposition.

Let's see this in action:

```python
preds = np.array(list(zip(np.random.normal(size=10),
                          np.array(np.random.normal(size=10, loc=2)))))
target = np.array(np.random.exponential(size=10))
```

```python
preds
```

```python
np.linalg.inv(preds.T.dot(preds)).dot(preds.T).dot(target)
```

```python
LinearRegression(fit_intercept=False).fit(preds, target).coef_
```


# Random Signals and LTI-Systems

*This jupyter notebook is part of a [collection of notebooks](../index.ipynb) on various topics of Digital Signal Processing. Please direct questions and suggestions to [Sascha.Spors@uni-rostock.de](mailto:Sascha.Spors@uni-rostock.de).*

## The Wiener Filter

The [Wiener filter](https://en.wikipedia.org/wiki/Wiener_filter), named after [*Nobert Wiener*](https://en.wikipedia.org/wiki/Norbert_Wiener), aims at estimating an unknown random signal by filtering a noisy distorted observation of the signal. It has a wide range of applications in signal processing. For instance, noise reduction, system identification, deconvolution and signal detection. The Wiener filter is frequently used as basic building block for algorithms that denoise audio signals, like speech, or remove noise from a image.

### Signal Model

The following signal model is underlying the Wiener filter

The random signal $s[k]$ is subject to distortion by the linear time-invariant (LTI) system $G(\mathrm{e}^{\,\mathrm{j}\,\Omega})$ and additive noise $n[k]$, resulting in the observed signal $x[k] = s[k] * g[k] + n[k]$. The additive noise $n[k]$ is assumed to be uncorrelated from $s[k]$. It is furthermore assumed that all random signals are wide-sense stationary (WSS). This distortion model holds for many practical problems, like e.g. the measurement of a physical quantity by a sensor.

The basic concept of the Wiener filter is to apply the LTI system $H(\mathrm{e}^{\,\mathrm{j}\,\Omega})$ to the observed signal $x[k]$ such that the output signal $y[k] = x[k] * h[k]$ matches $s[k]$ as close as possible. In order to quantify the deviation between $y[k]$ and $s[k]$, the error signal

\begin{equation}
e[k] = y[k] - s[k]
\end{equation}

is introduced. The error $e[k]$ equals zero, if the output signal $y[k]$ perfectly matches $s[k]$. In general, this goal cannot be achieved in a strict sense. As alternative one could aim at minimizing the linear average of the error $e[k]$. However, this quantity is not very well suited for optimization since it is signed. Instead, the quadratic average of the error $e[k]$ is used in the Wiener filter and other techniques. This measure is known as [*mean squared error*](https://en.wikipedia.org/wiki/Mean_squared_error) (MSE). It is defined as

\begin{equation}
E \{ |e[k]|^2 \} = \frac{1}{2 \pi} \int\limits_{-\pi}^{\pi} \Phi_{ee}(\mathrm{e}^{\,\mathrm{j}\,\Omega}) \,\mathrm{d}\Omega = \varphi_{ee}[\kappa] \Big\vert_{\kappa = 0}
\end{equation}

the equalities are deduced from the properties of the power spectral density (PSD) and the auto-correlation function (ACF). Above equation is referred to as [*cost function*](https://en.wikipedia.org/wiki/Loss_function). We aim at minimizing the cost function, hence minimizing the MSE between the original signal $s[k]$ and its estimate $y[k]$. The solution is referred to as [minimum mean squared error](https://en.wikipedia.org/wiki/Minimum_mean_square_error) (MMSE) solution.

### Transfer Function of the Wiener Filter

At first, the Wiener filter shall only have access to the observed signal $x[k]$ and selected statistical measures. It is assumed that the cross-power spectral density $\Phi_{xs}(\mathrm{e}^{\,\mathrm{j}\,\Omega})$ between the observed signal $x[k]$ and the original signal $s[k]$, and the PSD of the observed signal $\Phi_{xx}(\mathrm{e}^{\,\mathrm{j}\,\Omega})$ are known. This knowledge can either be gained by estimating both from measurements taken at an actual system or by using suitable statistical models.

The optimal filter is found by minimizing the MSE $E \{ |e[k]|^2 \}$ with respect to the transfer function $H(\mathrm{e}^{\,\mathrm{j}\,\Omega})$. The solution of this optimization problem goes beyond the scope of this notebook and can be found in the literature, e.g. [[Girod et. al](../index.ipynb#Literature)]. The transfer function of the Wiener filter is given as

\begin{equation}
H(\mathrm{e}^{\,\mathrm{j}\,\Omega}) = \frac{\Phi_{sx}(\mathrm{e}^{\,\mathrm{j}\,\Omega})}{\Phi_{xx}(\mathrm{e}^{\,\mathrm{j}\,\Omega})} = \frac{\Phi_{xs}(\mathrm{e}^{\,-\mathrm{j}\,\Omega})}{\Phi_{xx}(\mathrm{e}^{\,\mathrm{j}\,\Omega})}
\end{equation}

for $\Phi_{xx}(\mathrm{e}^{\,\mathrm{j}\,\Omega}) \neq 0$.
No knowledge on the actual distortion process is required besides the assumption of an LTI system and additive noise. Only the PSDs $\Phi_{sx}(\mathrm{e}^{\,\mathrm{j}\,\Omega})$ and $\Phi_{xx}(\mathrm{e}^{\,\mathrm{j}\,\Omega})$ have to be known in order to estimate $s[k]$ from $x[k]$ in the MMSE sense. Care has to be taken that the filter $H(\mathrm{e}^{\,\mathrm{j}\,\Omega})$ is causal and stable in practical applications.

**Example - Denoising of an Deterministic Signal**

The following example considers the estimation of the original signal from a distorted observation. It is assumed that the original signal is a deterministic signal $s[k] = \sin[\Omega_0\,k]$ which is distorted by an LTI system and additive normally distributed zero-mean white noise with PSD $N_0 = 0.1$. The PSDs $\Phi_{sx}(\mathrm{e}^{\,\mathrm{j}\,\Omega})$ and $\Phi_{xx}(\mathrm{e}^{\,\mathrm{j}\,\Omega})$ are estimated from $s[k]$ and $x[k]$ using the [Welch technique](../spectral_estimation_random_signals/welch_method.ipynb). The Wiener filter is applied to the observation $x[k]$ in order to compute the estimate $y[k]$ of $s[k]$. Note that the discrete signals have been illustrated by continuous lines for ease of illustration.

```python
%matplotlib inline
import numpy as np
import matplotlib.pyplot as plt
import scipy.signal as sig

N = 8129  # number of samples
M = 256  # length of Wiener filter
Om0 = 0.1*np.pi  # frequency of original signal
N0 = 0.1  # PSD of additive white noise

# generate original signal
s = np.cos(Om0 * np.arange(N))
# generate observed signal
g = 1/20*np.asarray([1, 2, 3, 4, 5, 4, 3, 2, 1])
np.random.seed(1)
n = np.random.normal(size=N, scale=np.sqrt(N0))
x = np.convolve(s, g, mode='same') + n
# estimate (cross) PSDs using Welch technique
f, Pxx = sig.csd(x, x, nperseg=M)
f, Psx = sig.csd(s, x, nperseg=M)
# compute Wiener filter
H = Psx/Pxx
H = H * np.exp(-1j*2*np.pi/len(H)*np.arange(len(H))*(len(H)//2))  # shift for causal filter
h = np.fft.irfft(H)
# apply Wiener filter to observation
y = np.convolve(x, h, mode='same')

# plot (cross) PSDs
Om = np.linspace(0, np.pi, num=len(H))

plt.figure(figsize=(10, 4))
plt.subplot(121)
plt.plot(Om, 20*np.log10(np.abs(.5*Pxx)), label=r'$| \Phi_{xx}(e^{j \Omega}) |$ in dB')
plt.plot(Om, 20*np.log10(np.abs(.5*Psx)), label=r'$| \Phi_{sx}(e^{j \Omega}) |$ in dB')
plt.title('(Cross) PSDs')
plt.xlabel(r'$\Omega$')
plt.legend()
plt.axis([0, np.pi, -60, 40])
plt.grid()

# plot transfer function of Wiener filter
plt.subplot(122)
plt.plot(Om, 20*np.log10(np.abs(H)))
plt.title('Transfer function of Wiener filter')
plt.xlabel(r'$\Omega$')
plt.ylabel(r'$| H(e^{j \Omega}) |$ in dB')
plt.axis([0, np.pi, -150, 3])
plt.grid()
plt.tight_layout()

# plot signals
idx = np.arange(500, 600)
plt.figure(figsize=(10, 4))
plt.plot(idx, x[idx], label=r'observed signal $x[k]$')
plt.plot(idx, s[idx], label=r'original signal $s[k]$')
plt.plot(idx, y[idx], label=r'estimated signal $y[k]$')
plt.title('Signals')
plt.xlabel(r'$k$')
plt.axis([idx[0], idx[-1], -1.5, 1.5])
plt.legend()
plt.grid()
```

**Exercise**

* Take a look at the PSDs and the resulting transfer function of the Wiener filter. How does the Wiener filter remove the noise from the observed signal?
* Change the frequency `Om0` of the original signal $s[k]$ and the noise power `N0` of the additive noise. What changes?

The cross-PSD $\Phi_{sx}(e^{j \Omega})$ has its maximum at the frequency $\Omega_0$ of the original signal $s[k]$, besides this frequency the PSD $\Phi_{xx}(e^{j \Omega})$ has approximately the value of the PSD $\Phi_{nn}(e^{j \Omega}) = N_0$ of the additive noise. The resulting Wiener filter $G(e^{j \Omega})$ has an attenuation of 0 dB at $\Omega_0$ which increases off this frequency. It has the characteristics of a band-pass filter centered at $\Omega_0$. By filtering the observed signal $x[k]$ with the Wiener filter, the major portion of the additive white noise is removed. However, around $\Omega_0$ both the original signal $s[k]$ and noise pass the filter. This explains the remaining small deviations between the estimated signal $y[k]$ and the original signal $y[k]$. The adaption of the Wiener filter to the original signal and additive noise changes when changing its frequency and PSD.

### Wiener Deconvolution

As discussed above, the general formulation of the Wiener filter is based on the knowledge of the PSDs $\Phi_{sx}(\mathrm{e}^{\,\mathrm{j}\,\Omega})$ and $\Phi_{xx}(\mathrm{e}^{\,\mathrm{j}\,\Omega})$ characterizing the distortion process and the observed signal, respectively. These PSDs can be derived from the PSDs of the original signal $\Phi_{ss}(\mathrm{e}^{\,\mathrm{j}\,\Omega})$ and the noise $\Phi_{nn}(\mathrm{e}^{\,\mathrm{j}\,\Omega})$, and the transfer function $G(\mathrm{e}^{\,\mathrm{j}\,\Omega})$ of the distorting system.

Under the assumption that $n[k]$ is uncorrelated from $s[k]$, the PSD $\Phi_{sx}(\mathrm{e}^{\,\mathrm{j}\,\Omega})$ can be derived as

\begin{equation}
\Phi_{sx}(\mathrm{e}^{\,\mathrm{j}\,\Omega}) = \Phi_{ss}(\mathrm{e}^{\,\mathrm{j}\,\Omega}) \cdot G(\mathrm{e}^{\,-\mathrm{j}\,\Omega})
\end{equation}

and

\begin{equation}
\Phi_{xx}(\mathrm{e}^{\,\mathrm{j}\,\Omega}) = \Phi_{ss}(\mathrm{e}^{\,\mathrm{j}\,\Omega}) \cdot  |G(\mathrm{e}^{\,\mathrm{j}\,\Omega})|^2 + \Phi_{nn}(\mathrm{e}^{\,\mathrm{j}\,\Omega})
\end{equation}

Introducing these results into the general formulation of the Wiener filter yields

\begin{equation}
H(\mathrm{e}^{\,\mathrm{j}\,\Omega}) = \frac{\Phi_{ss}(\mathrm{e}^{\,\mathrm{j}\,\Omega}) \cdot G(\mathrm{e}^{\,-\mathrm{j}\,\Omega})}{\Phi_{ss}(\mathrm{e}^{\,\mathrm{j}\,\Omega}) \cdot  |G(\mathrm{e}^{\,\mathrm{j}\,\Omega})|^2 + \Phi_{nn}(\mathrm{e}^{\,\mathrm{j}\,\Omega})}
\end{equation}

This specialization is also known as [*Wiener deconvolution filter*](https://en.wikipedia.org/wiki/Wiener_deconvolution). The filter can be derived from the PSDs of the original signal $s[k]$ and the noise $n[k]$, and the transfer function $G(\mathrm{e}^{\,\mathrm{j}\,\Omega})$ of the distorting system. This form is especially useful when the PSDs can be modeled reasonably well by analytic probabilty density functions (PSDs). In many cases, the additive noise can be modeled as white noise $\Phi_{nn}(e^{j \Omega}) = N_0$.

### Interpretation

The Wiener deconvolution filter derived above is also useful for an interpretation of the operation of the Wiener filter. It can be rewritten by introducing the frequency dependent [signal-to-noise ratio](https://en.wikipedia.org/wiki/Signal-to-noise_ratio) $\text{SNR}(\mathrm{e}^{\,\mathrm{j}\,\Omega}) = \frac{\Phi_{ss}(\mathrm{e}^{\,\mathrm{j}\,\Omega})}{\Phi_{nn}(\mathrm{e}^{\,\mathrm{j}\,\Omega})}$ between the original signal and the noise as

\begin{equation}
H(\mathrm{e}^{\,\mathrm{j}\,\Omega}) = \frac{1}{G(\mathrm{e}^{\,\mathrm{j}\,\Omega})} \cdot \left(   \frac{|G(\mathrm{e}^{\,\mathrm{j}\,\Omega})|^2}{|G(\mathrm{e}^{\,\mathrm{j}\,\Omega})|^2 + \frac{1}{\text{SNR}(\mathrm{e}^{\,\mathrm{j}\,\Omega})}} \right)
\end{equation}

Now two special cases are discussed:

1. If there is no additive noise $\Phi_{nn}(\mathrm{e}^{\,\mathrm{j}\,\Omega}) = 0$, the bracketed expression is equal to 1. Hence, the Wiener filter is simply given as the inverse system to the distorting system $$H(\mathrm{e}^{\,\mathrm{j}\,\Omega}) = \frac{1}{G(\mathrm{e}^{\,\mathrm{j}\,\Omega})}$$

2. If the distorting system is just a pass-through $G(\mathrm{e}^{\,\mathrm{j}\,\Omega}) = 1$, the Wiener filter is given as

    $$H(\mathrm{e}^{\,\mathrm{j}\,\Omega}) = \frac{\text{SNR}(\mathrm{e}^{\,\mathrm{j}\,\Omega})}{\text{SNR}(\mathrm{e}^{\,\mathrm{j}\,\Omega}) + 1}=\frac{\Phi_{ss}(\mathrm{e}^{\,\mathrm{j}\,\Omega})}{\Phi_{ss}(\mathrm{e}^{\,\mathrm{j}\,\Omega})+\Phi_{nn}(\mathrm{e}^{\,\mathrm{j}\,\Omega})}$$
    Two extreme cases are considered
    * for $\Phi_{ss}(\mathrm{e}^{\,\mathrm{j}\,\Omega})\gg \Phi_{nn}(\mathrm{e}^{\,\mathrm{j}\,\Omega})$ (high $\text{SNR}(\mathrm{e}^{\,\mathrm{j}\,\Omega})$) at a given frequency $\Omega$ the transfer function approaches 1
    * for $\Phi_{ss}(\mathrm{e}^{\,\mathrm{j}\,\Omega})\ll \Phi_{nn}(\mathrm{e}^{\,\mathrm{j}\,\Omega})$ (low $\text{SNR}(\mathrm{e}^{\,\mathrm{j}\,\Omega})$) at a given frequency $\Omega$ the transfer function approaches 0

The discussed cases explain the operation of the Wiener filter for specific situations, the general operation is a combination of these.

**Example - Denoising of a deterministic signal with the Wiener deconvolution filter**

The preceding example of the general Wiener filter will now be reevaluated with the Wiener deconvolution filter.

```python
N = 8129  # number of samples
M = 256  # length of Wiener filter
Om0 = 0.1*np.pi  # frequency of original signal
N0 = .1  # PSD of additive white noise

# generate original signal
s = np.cos(Om0 * np.arange(N))
# generate observed signal
g = 1/20*np.asarray([1, 2, 3, 4, 5, 4, 3, 2, 1])
np.random.seed(1)
n = np.random.normal(size=N, scale=np.sqrt(N0))
x = np.convolve(s, g, mode='same') + n
# estimate PSD
f, Pss = sig.csd(s, s, nperseg=M)
f, Pnn = sig.csd(n, n, nperseg=M)
# compute Wiener filter
G = np.fft.rfft(g, M)
H = 1/G * (np.abs(G)**2 / (np.abs(G)**2 + N0/Pss))
H = H * np.exp(-1j*2*np.pi/len(H)*np.arange(len(H))*(len(H)//2-8))  # shift for causal filter
h = np.fft.irfft(H)
# apply Wiener filter to observation
y = np.convolve(x, h, mode='same')

# plot (cross) PSDs
Om = np.linspace(0, np.pi, num=len(H))

plt.figure(figsize=(10, 4))
plt.subplot(121)
plt.plot(Om, 20*np.log10(np.abs(.5*Pss)), label=r'$| \Phi_{ss}(e^{j \Omega}) |$ in dB')
plt.plot(Om, 20*np.log10(np.abs(.5*Pnn)), label=r'$| \Phi_{nn}(e^{j \Omega}) |$ in dB')
plt.title('PSDs')
plt.xlabel(r'$\Omega$')
plt.legend()
plt.axis([0, np.pi, -60, 40])
plt.grid()

# plot transfer function of Wiener filter
plt.subplot(122)
plt.plot(Om, 20*np.log10(np.abs(H)))
plt.title('Transfer function of Wiener filter')
plt.xlabel(r'$\Omega$')
plt.ylabel(r'$| H(e^{j \Omega}) |$ in dB')
plt.axis([0, np.pi, -150, 10])
plt.grid()
plt.tight_layout()

# plot signals
idx = np.arange(500, 600)
plt.figure(figsize=(10, 4))
plt.plot(idx, x[idx], label=r'observed signal $x[k]$')
plt.plot(idx, s[idx], label=r'original signal $s[k]$')
plt.plot(idx, y[idx], label=r'estimated signal $y[k]$')
plt.title('Signals')
plt.xlabel(r'$k$')
plt.axis([idx[0], idx[-1], -1.5, 1.5])
plt.legend()
plt.grid()
```

**Exercise**

* Compare the transfer function of the Wiener deconvolution filter to the transfer function of the general Wiener filter from the preceding example.
* What is different compared to the general Wiener filter? Why?

The transfer function $H(e^{j \Omega})$ of the Wiener deconvolution filter is much smoother than the transfer function of the general Wiener filter from the preceding example. This is due to the knowledge of the transfer function $G(e^{j \Omega})$ of the disturbing system. The general Wiener filter inherently estimates this from the noisy observations.

**Copyright**

This notebook is provided as [Open Educational Resource](https://en.wikipedia.org/wiki/Open_educational_resources). Feel free to use the notebook for your own purposes. The text is licensed under [Creative Commons Attribution 4.0](https://creativecommons.org/licenses/by/4.0/), the code of the IPython examples under the [MIT license](https://opensource.org/licenses/MIT). Please attribute the work as follows: *Sascha Spors, Digital Signal Processing - Lecture notes featuring computational examples, 2016-2017*.


---
## 01. Transformada de Fourier

Eduard Larrañaga (ealarranaga@unal.edu.co)

---

### Resumen

En este cuaderno se presentan las transformadas de Fourier de una función.

---

Las transformadas de Fourier son una herramienta muy importante para entender, analizar suavizar y filtrar funciones o señales. También permiten resolver algunas ecuaciones diferenciales.

---
## Series de Fourier

Consideraremos  una función periodica $f(x)$ en un intervalo finito $0\leq x < L$. Si la función es par (simétrica) con respecto al punto medio $x = \frac{L}{2}$, la función puede expandirse en ua serie de cosenos con la forma

\begin{equation}
f(x) = \sum_{k=0}^{\infty} \alpha_k \cos \left( \frac{2\pi kx}{L} \right).
\end{equation}

Si la función es impar con respecto al punto medio del intervalo, se puede expandir en términos de funciones seno,

\begin{equation}
f(x) = \sum_{k=1}^{\infty} \beta_k \sin \left( \frac{2\pi kx}{L} \right).
\end{equation}

En los dos casos, los coeficientes $\alpha_k$ y $\beta_k$ son constantes que determinan las diferentes contribuciones en la superposición.

En el caso de una función general (sin una simetría especial), se puede realizar la expansión

\begin{equation}
f(x) = \sum_{k=0}^{\infty} \alpha_k \cos \left( \frac{2\pi kx}{L} \right) + \sum_{k=1}^{\infty} \beta_k \sin \left( \frac{2\pi kx}{L} \right)
\end{equation}

la cual puede re-escribirse en términos de funciones exponenciales complejas,

\begin{equation}
f(x) = \sum_{k=-\infty}^{\infty} \gamma_k \exp \left( i\frac{2\pi kx}{L} \right)
\end{equation}

donde

\begin{equation}
\gamma_k =
\begin{cases}
\frac{1}{2}(\alpha_{-k} + i \beta_{-k}) & \text{ si }k<0\\
\alpha_0 & \text{ si }k=0\\
\frac{1}{2}(\alpha_{k} - i \beta_{k}) & \text{ si }k>0
\end{cases}
\end{equation}

Cuando la función $f(x)$ no es periodica, es posible realizar la expansión en serie de Fourier considerando la función en el intervalo $0\leq x <L$ y repitiendo este comportamiento una y otra vez a izquierda y derecha. Sin embargo, la serie obtenida solo será util en el intervalo considerado.

---
### Coeficientes en la Serie de Fourier

Los coeficientes $\gamma_k$ son, en general, números complejos y pueden obtenerse mediante la expresión

\begin{equation}
\gamma_k = \frac{1}{L} \int_0^L f(x) \exp \left( -i\frac{2\pi kx}{L}\right) dx .
\end{equation}

## Transformada Discreta de Fourier (DFT)

A nivel computacional, la integral que define los coeficientes en la serie de Fourier debe realizarse numéricamente. Discretizando el intervalo de integración definiendo $n$ nodos, se tendran $N = n-1$ sub-intervalos con un tamaño $\Delta x = \frac{L}{N}$. Al utilizar la regla del trapezoide descrita en una clase anterior, se tiene

\begin{equation}
\gamma_k = \frac{1}{N}\left[ \frac{1}{2}f(0) + \frac{1}{2} f(L) + \sum_{j=1}^{N-1} f(x_j) \exp\left( -i \frac{2\pi kx_j}{L} \right) \right].
\end{equation}

Ya que la función debe ser periodica en el intervalo de integración , se tiene $f(0) = f(L)$ y con ello los dos primeros terminos se pueden incorporar dentro de la sumatoria,

\begin{equation}
\gamma_k = \frac{1}{N}\sum_{j=0}^{N-1} f(x_j) \exp\left( -i \frac{2\pi kx_j}{L} \right).
\end{equation}

Finalmente, debido a que los intervalos son iguales se puede escribir

\begin{equation}
x_j = \frac{j}{N}L
\end{equation}

Los coeficientes se pueden escribir en la forma

\begin{equation}
\gamma_k = \frac{1}{N}\sum_{j=0}^{N-1} f(x_j) \exp\left( -i \frac{2\pi kj}{N} \right).
\end{equation}

Aunque esta expresión yas corresponde a la transformada de Fourier, en muchas referencias bibliográficas se suele denominar *Transformada Discreta de Fourier (DFT)* a los coeficientes

\begin{equation}
c_k = \sum_{j=0}^{N-1} f_j \exp\left( -i \frac{2\pi kj}{N} \right)
\end{equation}

de tal forma que $c_k = N \gamma_k$ y donde se ha utilizado $f_j = f(x_j)$.

A partir de los coeficientes obtenidos se puede recuperar la discretización de la función. A esto se le denomina *Inversa de la Transformada Discreta de Fourier* y corresponde a la relación

\begin{equation}
f_j = \frac{1}{N} \sum_{k=0}^{N-1} c_k \exp\left( i \frac{2\pi kj}{N} \right).
\end{equation}

### DFT de una Función Real

Los resultados descritos arriba son aplicables a funciones reales o complejas. Sin embargo, en el caso de funciones reales el tiempo de computo de los coeficientes se puede reducir notablemente.

Para comprender la razón, considere el coeficiente $c_k$ para un índice $k$ menor que $N$ pero mayor que $\frac{1}{2}N$. Se puede re-escribir entonces $k = N - \alpha$ con $1\leq \alpha < \frac{1}{2}N$. Se tiene entonces que

\begin{align}
c_{N-\alpha} = &\sum_{j=0}^{N-1} f_j \exp\left( -i \frac{2\pi (N-\alpha )j}{N} \right)\\
c_{N-\alpha} = & \sum_{j=0}^{N-1} f_j \exp\left( -i 2\pi j \right) \exp\left( i \frac{2\pi \alpha j}{N} \right) \\
c_{N-\alpha} = & \sum_{j=0}^{N-1} f_j  \exp\left( i \frac{2\pi \alpha j}{N} \right)\\
c_{N-\alpha} = & c^*_\alpha,
\end{align}

donde $^*$ signifca complejo conjugado y se ha utilizado la suposiciónd e una función real, i.e. $f^*(x) = f(x)$. Esta relación quiere decir que

\begin{align}
c_{N-1} = &c^*_1 \\
c_{N-2} = &c^*_2 \\
c_{N-3} = &c^*_3 \\
...
\end{align}

Se concluye entonces que, para funciones reales, solo se deben calcular los coeficientes  $c_k$  para $0\leq k \leq \frac{1}{2}N$ y los demás quedan automáticamente definidos.

Cuando $N$ es par, se requiere calcular $\frac{1}{2}N + 1$ coeficientes, mientras que si $N$ es impar se requiere calcular $\frac{1}{2} (N+1)$ coeficientes.

### Ejemplo

En el archivo `signal.txt` se encuentran los datos de una señal para la cual calcularemos su DFT.

```python
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline

# Reading the data
f_data = np.loadtxt( 'signal.txt')
x_data = np.arange(len(f_data))
plt.figure(figsize=(10,4))
plt.plot(x_data, f_data, color='black')
plt.title('')
plt.xlabel(r'$x$')
plt.ylabel(r'$f(x)$')
plt.grid()
plt.show()
```

El número total de datos es

```python
n = len(f_data)
n
```

    1024

```python
c = np.zeros(n//2+1, complex)
for k in range(n//2+1):
    for i in range(n):
        c[k] += f_data[i]*np.exp(-2j*np.pi*k*i/n)

```

```python
plt.figure(figsize=(10,7))
plt.plot(range(len(c)), abs(c), color='black')
plt.title('')
plt.xlabel(r'$k$')
plt.ylabel(r'$\left| c_k \right|$')
plt.title(r'Transformada Discreta de Fourier (DFT)')
plt.grid()
plt.show()
```

## Transformada de Fourier 2-Dimensional

El desarrollo presentado se puede extender fácilmente al caso de funciones 2-dimensionales, $f(x,y)$. Para ello se realiza la DFT con respecto a una de las variables y luego con respecto a la otra variable de forma independiente.

Considere la función

\begin{equation}
f_{\mu, \nu} = f(x_\mu, y_\nu)
\end{equation}

con $\mu = 0, 1, 2, ..., m-1$ y $\nu = 0, 1, 2, ..., n-1$. Primero se realiza la transformada sobre cada una de $\mu$,

\begin{equation}
c'_{\mu, \beta} = \sum_{\nu=0}^{N-1}f_{\mu,\nu} \exp \left( -i\frac{2\pi \beta \nu}{N} \right),
\end{equation}

donde $N=n-1$. De esta forma, para cada $\mu$ se tienen $N$ coeficientes etiquetados con el índice $\beta$.

Ahora se realiza la DFT para cada valor de $\beta$,

\begin{equation}
c_{\alpha, \beta} = \sum_{\mu=0}^{M-1}c'_{\mu,\beta} \exp \left( -i\frac{2\pi \alpha \mu}{M} \right),
\end{equation}

donde $M=m-1$. Al unir estas dos ecuaciones se obtiene la transformada 2-dimensional

\begin{equation}
c_{\alpha, \beta} = \sum_{\mu=0}^{M-1}\sum_{\nu=0}^{N-1}f_{\mu,\nu} \exp \left( -i\frac{2\pi \beta \nu}{N} \right) \exp \left( -i\frac{2\pi \alpha \mu}{M} \right)
\end{equation}

\begin{equation}
c_{\alpha, \beta} = \sum_{\mu=0}^{M-1}\sum_{\nu=0}^{N-1}f_{\mu,\nu} \exp \left[ -i2\pi \left(   \frac{ \alpha \mu}{M} + \frac{ \beta \nu}{N} \right) \right]
\end{equation}

De forma similar, la DFT inversa resulta ser

\begin{equation}
f_{\mu, \nu} = \frac{1}{MN} \sum_{\alpha=0}^{M-1}\sum_{\beta=0}^{N-1} c_{\alpha,\beta} \exp \left[ i2\pi \left(   \frac{ \alpha \mu}{M} + \frac{ \beta \nu}{N} \right) \right]
\end{equation}

---
## Transformada Rápida de Fourier (FFT)

Para realizar la DFT,

\begin{equation}
c_k = \sum_{j=0}^{N-1} f_j \exp\left( -i \frac{2\pi kj}{N} \right)
\end{equation}

se requiere calcular $\frac{1}{2} (N+1)$ coeficientes y en cada uno de ellos se deben sumar $N$ términos. Por ello el número total de operaciones es $\frac{1}{2} (N+1)N \sim \mathcal{O} \left( \frac{1}{2} n
N^2\right)$.

Para mejorar el tiempo de cálculo, se puede optimizar el proceso introduciendo la *Transformada Rápida de Fourier (FFT)*. Para compredner este procedimiento supondremos que la muestra tiene $N=2^m$ datos. Como $N$ es una potencia de $2$ es posible dividir la muestra en dos sub-conjuntos de igual tamaño: aquellos con $j$ par y aquellos con $j$ impar.

Para los datos pares, escribimos $j=2\alpha$ con $\alpha = 0,1,2,..,\frac{1}{2}N$ y con ello, la DFT toma la forma

\begin{equation}
E_k = \sum_{\alpha=0}^{\frac{1}{2}N-1} f_{2\alpha} \exp\left( -i \frac{2\pi k(2\alpha )}{N} \right) = \sum_{\alpha=0}^{\frac{1}{2}N-1} f_{2\alpha} \exp\left( -i \frac{2\pi k\alpha}{\frac{1}{2}N} \right)
\end{equation}

que corresponde simplemente a una DFT con $\frac{1}{2}N$ datos.

De forma similar, para los valores impares se escribe $j=2\alpha +1$ y al DFT toma la forma

\begin{equation}
\sum_{\alpha=0}^{\frac{1}{2}N-1} f_{2\alpha+1} \exp\left( -i \frac{2\pi k(2\alpha +1)}{N} \right) = \exp\left( -i \frac{2\pi k}{N} \right) \sum_{\alpha=0}^{\frac{1}{2}N-1} f_{2\alpha+1} \exp\left( -i \frac{2\pi k\alpha}{\frac{1}{2}N} \right) = \exp\left( -i \frac{2\pi k}{N} \right) O_k
\end{equation}

donde $O_k$ es una DFT con $\frac{1}{2}N$ datos.

EL coeficiente de Fourier Completo se obtiene al sumar términos pares e impares, i.e.

\begin{equation}
c_k =E_k + \exp\left( -i \frac{2\pi k}{N} \right) O_k
\end{equation}

De esta expresión se puede notar que, para calcular el coeficiente $c_k$ en la DFT, se reuquiere calcular dos términos $E_k$ y $O_k$ que son dos DFT mas pequeños (con la mitad de los datos) y un factor (twiddle factor) que es trivial de calcular.
Este proceso de reducción puede repetirse nuevamente al dividir cada uno de los coeficientes en dos DFT mas pequeñas y un (twiddle) factor adicional. El proceso de sub-división se repite hasta que se llega al momento en el que cada DFT es la transformada de un solo dato. El coeficiente de Fourier correspondiente es simplemente

\begin{equation}
c_0 = \sum_{j=0}^{0} f_0 e^0 = f_0,
\end{equation}

es decir solo un término!!!

Para comprender porque se mejora el tiempo de calculo con este procedimiento, debemos notar que para descomponer completamente el conjunto de $2^m$ datos se requieren $m=\log_2 N$ niveles de división. Por lo tanto, el número de coeficientes que se deben calcular en todo el procesos es de $N$ en cada uno de los $\log_2 N$ niveles y por lo tanto el número total de operaciones involucradas es de $\mathcal{O} \left( N \log_2 N\right)$ el cual es mucho mejor que el orden $\mathcal{O} \left( \frac{1}{2} N^2\right)$ de la DFT.

Por ejemplo, para una muestra de $N= 10^6$ datos la DFT requiere $\mathcal{O} \left( \frac{1}{2} N^2\right) \sim 5 \times 10^{11}$ operaciones mientras que la FFT requiere  $\mathcal{O} \left( N \log_2 N\right) \sim 2 \times 10^7$ operaciones.

---
En python se tienen varias implementaciones de la FFT. Por ejemplo están las funciones

- [scipy.fft](https://docs.scipy.org/doc/scipy/tutorial/fft.html)
- [numpy.fft](https://numpy.org/doc/stable/reference/routines.fft.html)

```python
from scipy.fft import fft

C = fft(f_data)
len(C)
```

    1024

```python
plt.figure(figsize=(10,7))
plt.plot(range(len(C)), abs(C), color='black')
plt.title('')
plt.xlabel(r'$k$')
plt.ylabel(r'$\left| C_k \right|$')
plt.title(r'Transformada Rápida de Fourier (FFT)')
plt.grid()
plt.show()
```

---
## Filtrado de una Señal

Primero leeremos la señal cruda (señal+ruido) y la visualizamos

```python
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline

# Reading the data
t, signal = np.loadtxt('synth_signal.txt', unpack=True)
clean_signal = np.loadtxt( 'clean_synth_signal.txt')
#minsignal, maxsignal = signal.min(), signal.max()

plt.figure(figsize=(12,7))
plt.plot(t, signal, color='cornflowerblue', label='Señal+Ruido')
#plt.plot(t, clean_signal, color='crimson', label='Señal')
plt.title('')
plt.xlabel(r'$t$')
plt.ylabel(r'$f(t)$')
plt.legend()
plt.grid()
plt.show()
```

La FFT de esta señal es

```python
## Fast Fourier Transform
n = len(t)
c = np.fft.fft(signal, n) #computes the fft
dt = t[1] - t[0]
psd = c*np.conj(c)/n
freq = (1/(dt*n)) * np.arange(n) #frequency array

plt.figure(figsize=(12,7))
plt.plot(freq, abs(psd), color='cornflowerblue')
plt.title('')
plt.xlabel(r'frequency')
plt.ylabel(r'$\left| C_k \right|$')
plt.grid()
plt.show()
```

```python
## Filter out noise
threshold = 100
psd_idxs = psd > threshold # array of 0s and 1s
psd_clean = psd * psd_idxs # zero out all the unnecessary powers
c_clean = psd_idxs * c # used to retrieve the signal

signal_filtered = np.fft.ifft(c_clean) #inverse fourier transform
```

```python
plt.figure(figsize=(12,5))
plt.plot(t, np.real(signal_filtered), color='crimson')
plt.ylim(-4,4)
plt.title('')
plt.xlabel(r'$t$')
plt.ylabel(r'$ f_k $')
plt.grid()
plt.show()
```

```python
plt.figure(figsize=(12,7))
#plt.plot(t, signal, color='cornflowerblue', label='Señal+Ruido')
plt.plot(t, np.real(signal_filtered), color='crimson', label='Señal recuperada')
plt.plot(t, clean_signal, color='green', label='Señal original')
plt.title('')
plt.ylim(-7,7)
plt.xlabel(r'$t$')
plt.ylabel(r'$f(t)$')
plt.legend()
plt.grid()
plt.show()
```

```python
## Create the synthetic signal
dt = 0.001
t = np.arange(0, 1, dt)
signal = np.sin(2*np.pi*50*t) + np.sin(2*np.pi*120*t) # composite signal
signal_clean = signal # copy for later comparison
signal = signal + 2.5 * np.random.randn(len(t))

data = np.column_stack((t,signal))
np.savetxt('synth_signal.txt',data)
data = np.column_stack((t,signal_clean))
np.savetxt('clean_synth_signal.txt',signal_clean)
```

```python

```

05. Transformada Fourier/01. Transformada de Fourier.ipynb


# Exercises

## Exercise 1: Refactor functions into a more general class

The section [Simple model problems and their solutions](#fem:deq:1D:models:simple) lists three functions for
computing the analytical solution of some simple model problems. There
is quite some repetitive code, suggesting that the functions can
benefit from being refactored into a class hierarchy, where the super
class solves $-(a(x)u'(x))'=f(x)$ and where subclasses define the
equations for the boundary conditions in a model. Make a method for
returning the residual in the differential equation and the boundary
conditions when the solution is inserted in these equations. Create a
test function that verifies that all three residuals vanish for each
of the model problems in the section [Simple model problems and their solutions](#fem:deq:1D:models:simple).  Also
make a method that returns the solution either as `sympy` expression
or as a string in LaTeX format.  Add a fourth subclass for the problem
$-(au')'=f$ with a Robin boundary condition:

$$
u(0)=0,\quad -u'(L) = C(u - D){\thinspace .}
$$

Demonstrate the use of this subclass for the case $f=0$ and $a=\sqrt{1+x}$.

**Solution.**
This is an exercise in software engineering.
The model-specific information is related to the boundary
conditions only. We can then let the super class take care of the
differential equation and the solution process, while subclasses
provide a method `get_bc` to
return the symbolic expressions for the boundary equations.

The super class may be coded as
shown below.

```python
import sympy as sym
x, L, C, D, c_0, c_1, = sym.symbols('x L C D c_0 c_1')

class TwoPtBoundaryValueProblem(object):
    """
    Solve -(a*u')' = f(x) with boundary conditions
    specified in subclasses (method get_bc).
    a and f must be sympy expressions of x.
    """
    def __init__(self, f, a=1, L=L, C=C, D=D):
        """Default values for L, C, D are symbols."""
        self.f = f
        self.a = a
        self.L = L
        self.C = C
        self.D = D

        # Integrate twice
        u_x = - sym.integrate(f, (x, 0, x)) + c_0
        u = sym.integrate(u_x/a, (x, 0, x)) + c_1
        # Set up 2 equations from the 2 boundary conditions and solve
        # with respect to the integration constants c_0, c_1
        eq = self.get_bc(u)
        eq = [sym.simplify(eq_) for eq_ in eq]
        print(('BC eq:', eq))
        self.u = self.apply_bc(eq, u)

    def apply_bc(self, eq, u):
        # Solve BC eqs respect to the integration constants
        r = sym.solve(eq, [c_0, c_1])
        # Substitute the integration constants in the solution
        u = u.subs(c_0, r[c_0]).subs(c_1, r[c_1])
        u = sym.simplify(sym.expand(u))
        return u

    def get_solution(self, latex=False):
        return sym.latex(self.u, mode='plain') if latex else self.u

    def get_residuals(self):
        """Return the residuals in the equation and BCs."""
        R_eq = sym.diff(sym.diff(self.u, x)*self.a, x) + self.f
        R_0, R_L = self.get_bc(self.u)
        residuals = [sym.simplify(R) for R in (R_eq, R_0, R_L)]
        return residuals

    def get_bc(self, u):
        raise NotImplementedError(
            'class %s has not implemented get_bc' %
            self.__class__.__name__)
```

The various subclasses deal with the boundary conditions of the
various model problems:

```python
class Model1(TwoPtBoundaryValueProblem):
    """u(0)=0, u(L)=D."""
    def get_bc(self, u):
        return [u.subs(x, 0)-0,               # x=0 condition
                u.subs(x, self.L) - self.D]   # x=L condition

class Model2(TwoPtBoundaryValueProblem):
    """u'(0)=C, u(L)=D."""
    def get_bc(self, u):
        return [sym.diff(u,x).subs(x, 0) - self.C, # x=0 cond.
                u.subs(x, self.L) - self.D]        # x=L cond.

class Model3(TwoPtBoundaryValueProblem):
    """u(0)=C, u(L)=D."""
    def get_bc(self, u):
        return [u.subs(x, 0) - self.C,
                u.subs(x, self.L) - self.D]
```

A suitable test function gets quite compact:

```python
def test_TwoPtBoundaryValueProblem():
    f = 2
    model = Model1(f)
    print(('Model 1, u:', model.get_solution()))
    for R in model.get_residuals():
        assert R == 0

    f = x
    model = Model2(f)
    print(('Model 2, u:', model.get_solution()))
    for R in model.get_residuals():
        assert R == 0

    f = 0
    a = 1 + x**2
    model = Model3(f, a=a)
    print(('Model 3, u:', model.get_solution()))
    for R in model.get_residuals():
        assert R == 0
```

The fourth model is just about defining the boundary conditions as equations:

```python
class Model4(TwoPtBoundaryValueProblem):
    """u(0)=0, -u'(L)=C*(u-D)."""
    def get_bc(self, u):
        return [u.subs(x, 0) - 0,
                -sym.diff(u, x).subs(x, self.L) -
                self.C*(u.subs(x, self.L) - self.D)]
```

A demo function goes like

```python
def demo_Model4():
    f = 0
    model = Model4(f, a=sym.sqrt(1+x))
    print(('Model 4, u:', model.get_solution()))
```

The printout shows that the solution is

$$
u(x) = \frac{2CD\sqrt{1+L}(\sqrt{1+x}-1}{2C\sqrt{1+L} + 2C+1}{\thinspace .}
$$

Filename: `uxx_f_sympy_class`.

## Exercise 2: Compute the deflection of a cable with sine functions

A hanging cable of length $L$
with significant tension $T$ has a deflection $w(x)$
governed by

$$
T w''(x) = \ell(x),
$$

where $\ell(x)$ the vertical load per unit length.
The cable is fixed at $x=0$ and $x=L$ so the boundary conditions become
$w(0)=w(L)=0$. The deflection $w$ is positive upwards, and $\ell$ is
positive when it acts downwards.

If we assume a constant load $\ell(x)=\hbox{const}$,
the solution is expected to be symmetric around $x=L/2$. For a function
$w(x)$ that is symmetric around some point $x_0$, it means that
$w(x_0-h) = w(x_0+h)$, and then $w'(x_0)=\lim_{h\rightarrow 0}(w(x_0+h)-
w(x_0-h))/(2h)=0$. We can therefore utilize symmetry to halve the domain.
We then seek $w(x)$ in $[0,L/2]$ with boundary conditions $w(0)=0$ and
$w'(L/2)=0$.

The problem can be scaled by introducing dimensionless  variables,

$$
\bar x = \frac{x}{L/2},\quad \bar u = \frac{w}{w_c},
$$

where $w_c$ is a characteristic size of $w$.
Inserted in the problem for $w$,

$$
\frac{4Tw_c}{L^{2}}\frac{d^2\bar u}{d\bar x^2} = \ell\ (= \hbox{const}){\thinspace .}
$$

A desire is to have $u$ and its derivatives about unity, so
choosing $w_c$ such that $|d^2\bar u/d\bar x^2|=1$ is an idea.
Then $w_c=\frac{1}{4}\ell L^2/T$, and the problem for the scaled vertical
deflection $u$ becomes

$$
u'' = 1,\quad x\in (0,1),\quad u(0)=0,\ u'(1)=0{\thinspace .}
$$

Observe that there are no physical parameters in this scaled problem.
From now on we have for convenience
renamed $x$ to be the scaled quantity $\bar x$.

**a)**
Find the exact solution for the deflection $u$.

**Solution.**
[Exercise 1: Refactor functions into a more general class](#fem:deq:exer:BVP1D:class) or
the section [Simple model problems and their solutions](#fem:deq:1D:models:simple) features tools for finding
the analytical solution of this differential equation.
The present
model problem is close to model 2
in the section [Simple model problems and their solutions](#fem:deq:1D:models:simple). We can modify the `model2`
function:

```python
def model():
    """Solve u'' = -1, u(0)=0, u'(1)=0."""
    import sympy as sym
    x, c_0, c_1, = sym.symbols('x c_0 c_1')
    u_x = sym.integrate(1, (x, 0, x)) + c_0
    u = sym.integrate(u_x, (x, 0, x)) + c_1
    r = sym.solve([u.subs(x,0) - 0,
                   sym.diff(u,x).subs(x, 1) - 0],
                  [c_0, c_1])
    u = u.subs(c_0, r[c_0]).subs(c_1, r[c_1])
    u = sym.simplify(sym.expand(u))
    return u
```

The solution becomes

$$
u(x) = \frac{1}{2}x(x-2){\thinspace .}
$$

Plotting $u(x)$ shows that $|u|\in [0,\frac{1}{2}]$ which is compatible with
the aim of the scaling, i.e., to have $u$ of size *about* unity (at least
not very small or very large).

**b)**
A possible function space is spanned by ${\psi}_i=\sin ((2i+1)\pi x/2)$,
$i=0,\ldots,N$. These functions
fulfill the necessary condition ${\psi}_i(0)=0$,
but they also fulfill ${\psi}_i'(1)=0$ such that both boundary
conditions are fulfilled by the expansion $u=\sum_jc_j{\varphi}_j$.

Use a Galerkin and a least squares method to find the coefficients
$c_j$ in $u(x)=\sum_j c_j{\psi}_j$. Find how fast the coefficients
decrease in magnitude by looking at $c_j/c_{j-1}$.
Find the error in the maximum deflection at $x=1$ when only one
basis function is used ($N=0$).

**Hint.**
In this case, where the basis functions and their derivatives are
orthogonal, it is easiest to set up the calculations by hand and
use `sympy` to help out with the integrals.

**Solution.**
With $u=\sum_{j=0}^Nc_j{\psi}_j(x)$ the residual becomes

$$
R = 1 - u'' = 1 +\sum_{j=0}^Nc_j{\psi}_j''(x) =
1 + \sum_{j=0}^Nc_j(2j+1)^2\frac{\pi^2}{4}\sin((2j+1)\frac{\pi x}{2}){\thinspace .}
$$

**Least squares method.**
The minimization of $\int_0^1R^2dx$ leads to the equations

$$
(R,\frac{\partial R}{\partial c_i})=0,\quad i=0,\ldots,N{\thinspace .}
$$

We find that

$$
\frac{\partial R}{\partial c_i} =
(2i+1)^2\frac{\pi^2}{4}\sin((2i+1)\frac{\pi x}{2}),
$$

so the governing equations become

$$
(1+\sum_{j=0}^Nc_j(2j+1)^2\frac{\pi^2}{4}\sin((2j+1)\frac{\pi x}{2}),
(2i+1)^2\frac{\pi^2}{4}\sin((2i+1)\frac{\pi x}{2}) = 0{\thinspace .}
$$

By linearity of the inner product (or integral) this expression can
be reordered to

$$
\begin{align*}
\sum_{j=0}^Nc_j((2j+1)^2\frac{\pi^2}{4}\sin((2j+1)\frac{\pi x}{2}), &
(2i+1)^2\frac{\pi^2}{4}\sin((2i+1)\frac{\pi x}{2}) = \\
& -(1, (2i+1)^2\frac{\pi^2}{4}\sin((2i+1)\frac{\pi x}{2})),
\end{align*}
$$

which is nothing but a linear system

$$
\sum_{j=0}^N A_{i,j}c_j = b_i,\quad i=0,\ldots,N,
$$

with

$$
\begin{align*}
A_{i,j} &= (2j+1)^4\frac{\pi^4}{16}\int_0^1
\sin((2j+1)\frac{\pi x}{2})\sin((2i+1)\frac{\pi x}{2})dx,\\
b_i &= -(2i+1)^2\frac{\pi^2}{4}\int_0^1 \sin((2i+1)\frac{\pi x}{2})dx
\end{align*}
$$

Orthogonality of the sine functions $\sin (k\pi x/2)$ on $[0,1]$
for integer $k$ implies that
$A_{i,j}=0$ for $i\neq j$, and $A_{i,i}$ can be
computed by `sympy`:

```python
from sympy import *
i = symbols('i', integer=True)
x = symbols('x', real=True)
integrate(sin(i*pi*x/2)**2, (x, 0, 1))
```

Therefore,

$$
A_{i,j} = \left\lbrace\begin{array}{ll}
0,& i\neq j\\
\frac{1}{2} (2i+1)^4\frac{\pi^4}{16}, & i = j
\end{array}\right.
$$

The right-hand side can also be computed by `sympy`:

```python
integrate(sin((2*i+1)*pi*x/2), (x, 0, 1))
```

One should always be skeptical to symbolic software and integration of
periodic functions like the sine and cosine since the answers can be
too simplistic (see subexercise d!).
A general test is to perform numerical integration with
lots of sampling points to (partially) verify the symbolic formula.
Here is an application of the midpoint rule:

```python
def midpoint_rule(f, M=100000):
    """Integrate f(x) over [0,1] using M intervals."""
    from numpy import sum, linspace
    dx = 1.0/M                       # interval length
    x = linspace(dx/2, 1-dx/2, M)    # integration points
    return dx*sum(f(x))

def check_integral_b():
    from numpy import pi, sin
    for i in range(12):
        exact = 2/(pi*(2*i+1))
        numerical = midpoint_rule(
            f=lambda x: sin((2*i+1)*pi*x/2))
        print((i, abs(exact - numerical)))
```

The output shows that the difference between numerical and exact
integration is about $10^{-11}$, which is "small" (and gets smaller
by just increasing `M`). This result brings evidence that the
`sympy` answer is correct.
Alternatively, in this simple case, we can easily calculate the anti-derivative.
It goes like

$$
-\frac{2}{\pi(2i+1)}\cos((2k+1)\frac{\pi x}{2}),
$$

and for $x=1$ we get
$\cos\frac{\pi}{2}$, $\cos 3\frac{\pi}{2}$, $\cos 5\frac{\pi}{2}$,
and so on, which all evaluates to zero, and since the cosine is 1 for $x=0$,
the formula found by `sympy` is correct.

We then get

$$
b_i = -(2i+1)^2\frac{\pi^2}{4}\frac{2}{\pi(2i+1)} = -\frac{1}{2} (2i+1)\pi,
$$

and consequently,

$$
c_i = \frac{b_i}{A_{i,i}} = -\frac{\frac{1}{2} (2i+1)\pi}{\frac{1}{2} (2i+1)^4}\frac{\pi^4}{16} = -\frac{16}{\pi^3(2i+1)^3}{\thinspace .}
$$

**Galerkin's method.**
The Galerkin method applied to this problem
starts with

$$
(u'',v) = (1,v)\quad \forall v\in V,
$$

and the requirement that $v(0)=0$ since $u(0)=0$.
Integration by parts and using $u'(1)=0$ and $v(0)=0$ makes the boundary
term vanish, and the variational form becomes

$$
(u',v') = -(1,v) \quad \forall v\in V{\thinspace .}
$$

Inserting $u=\sum_{j=0}^Nc_j{\psi}_j(x)$ and $v={\psi}_i$ leads to

$$
\sum_{j=0}^N ({\psi}_j', {\psi}_i')c_j = (1,{\psi}_i),\quad i=0,\ldots,N{\thinspace .}
$$

With ${\psi}_i=\sin((2i+1)\frac{\pi x}{2})$ the matrix entries become

$$
A_{i,j} =  (2i+1)(2j+1)\frac{\pi^2}{4}\int_0^1 \cos((2i+1)\frac{\pi x}{2})
\cos((2j+1)\frac{\pi x}{2})dx{\thinspace .}
$$

Orthogonality of the cosine functions implies $A_{i,j}=0$ for
$i\neq j$, and $A_{i,i}$ is computed by integrating the square
of the cosine function,

```python
integrate(cos((k+1)*pi*x/2)**2, (x, 0, 1))
```

Now,

$$
A_{i,i} = (2i+1)^2\frac{\pi^2}{4}\frac{1}{2} = \frac{1}{8}(2i+1)^2 \pi^2{\thinspace .}
$$

The right-hand side has almost the same integral as in the
least squares case,

$$
b_i = -\int_0^1 \sin((2i+1)\frac{\pi x}{2})dx = -\frac{2}{\pi (2i+1)}{\thinspace .}
$$

Consequently,

$$
c_i = \frac{b_i}{A_{i,i}} = -\frac{16}{\pi^3(2i+1)^3},
$$

which is the same result as we obtained in the least squares method.

**Decay of coefficients.**
The coefficients decay,

$$
\frac{c_i}{c_{i+1}} = \left(\frac{2i+3}{2i+1}\right)^3 > 0{\thinspace .}
$$

The decay is most pronounced for the first terms:

```python
for i in range(10):
  print((float(2*i+3)/(2*i+1))**3)
```

**Error in one-term solution.**
Keeping just one term ($N=0$) means that

$$
u(x) = -\frac{16}{\pi^3}\sin(\frac{\pi x}{2}){\thinspace .}
$$

The maximum deflection at $x=1$ becomes $-16\pi^{-3}=-0.5160$, to be compared
with the exact value $-\frac{1}{2}$. The error is 3.2 percent.

**c)**
Visualize the solutions in b) for $N=0,1,20$.

**Solution.**
First we need a function to compute the approximate $u$ in this case:

```python
def sine_sum(x, N):
    s = 0
    from numpy import pi, sin, zeros
    u = [] # u[k] is the sum i=0,...,k
    k = 0
    for i in range(N+1):
        s += - 16.0/((2*i+1)**3*pi**3)*sin((2*i+1)*pi*x/2)
        u.append(s.copy())  # important with copy!
    return u
```

Note the need to append `s.copy()`: doing just `u.append(s)` will
make, e.g., `u[0]` a reference to `s`, which at the end of the
loop is an array corresponding to the maximum $i$ value.

We also need a function that can create an appropriate plot:

```python
def plot_sine_sum():
    from numpy import linspace
    x = linspace(0, 1, 501)  # coordinates for plot
    u = sine_sum(x, N=10)
    u_e = 0.5*x*(x-2)
    N_values = 0, 1, 10
    for k in N_values:
        plt.plot(x, u[k])
    plt.plot(x, u_e)
    plt.legend(['N=%d' % k for k in N_values] + ['exact'],
               loc='upper right')
    plt.xlabel('$x$');  plt.ylabel('$u$')
    plt.savefig('tmpc.png'); plt.savefig('tmpc.pdf')
```

The plot shows that the solution for $N=0$ has a slight deviation from the
exact curve, but even $N=1$ catches up visually with the exact solution (!).

**d)**
The functions in b) were selected such that they fulfill the
condition ${\psi}'(1)=0$. However, in the Galerkin method, where we
integrate by parts, the condition $u'(1)=0$ is incorporated in the
variational form. This leads to the idea of just choosing a simpler
basis, namely "all" sine functions ${\psi}_i = \sin((i+1)\frac{\pi x}{2})$.
Will the method adjust the coefficient such that the additional
functions compared with those in b) get vanishing coefficients? Or
will the additional basis functions improve the solution?
Use Galerkin's method.

**Solution.**
According to the calculations in b), the Galerkin method, with
${\psi}_i = \sin((i+1)\frac{\pi x}{2})$, leads to the almost the
same
matrix entries on the diagonal:

$$
\begin{align*}
A_{i,i} &= (i+1)(j+1)\frac{\pi^2}{4}\int_0^1 \cos((i+1)\frac{\pi x}{2})
\cos((j+1)\frac{\pi x}{2})dx\\
&= (i+1)^2\frac{\pi^2}{4}\frac{1}{2} = \frac{1}{8}(i+1)^2 \pi^2{\thinspace .}
\end{align*}
$$

The right-hand side becomes (as before)

$$
b_i = -\int_0^1 \sin((i+1)\frac{\pi x}{2})dx = -\frac{2}{\pi (i+1)}{\thinspace .}
$$

We may use `sympy` to integrate,

```python
integrate(sin((i+1)*pi*x/2), (x, 0, 1))
```

As noted in b), let us be a bit skeptical to this answer and check it.
A quick check with numerical integration,

```python
def check_integral_d_sympy_answer():
    from numpy import pi, sin
    for i in range(12):
        exact = 2/(pi*(i+1))
        numerical = midpoint_rule(
            f=lambda x: sin((i+1)*pi*x/2))
        print((i, abs(exact - numerical)))
```

gives the output

        0 6.54487575247e-12
        1 0.31830988621
        2 1.96350713466e-11
        3 0.159154943092
        4 3.27249061183e-11
        5 0.106103295473
        6 4.58150045679e-11
        7 0.0795774715459
        8 5.89047144395e-11
        9 0.0636619773677
        10 7.19949447281e-11
        11 0.0530516476973

It is clear that for $i$ odd, there are significant differences between
the `sympy` answer and the midpoint rule with high resolution!

We therefore need to do hand calculations to investigate this problem
further.
The anti-derivative is very easy to realize in this case:

$$
\begin{align*}
\int_0^1\sin ((i+1)\pi x/2)dx &= -\frac{2}{\pi(i+1)}(\cos((i+1)\frac{\pi}{2}) - \cos(0))\\
&= \frac{2}{\pi(i+1)}(1 - \cos((i+1)\frac{\pi}{2})){\thinspace .}
\end{align*}
$$

The value of the cosine expression depends on $i$, and the first values are

$i=0$	 $i=1$	 $i=2$	 $i=3$

   0        	    -1       	    0        	    1

This pattern repeats and is the same for four consecutive values of $i$.
Hence, the integral is $2/(\pi (i+1))$ for even $i$ ($i=2k$ for
integer $k$, or equivalently: when $i\mbox{ mod } 2 = 0$). For $i=4k+1$, or
equivalently: when $(i-1)\mbox{ mod } 4 = 0$, the
integral is $4/(\pi(4k+1))$, while for $i=4k+3$, the integral vanishes.
This is a more complicated answer than what `sympy` provides!

We can check our new answers against numerical integration:

```python
def check_integral_d():
    from numpy import pi, sin
    for i in range(24):
        if i % 2 == 0:
            exact = 2/(pi*(i+1))
        elif (i-1) % 4 == 0:
            exact = 2*2/(pi*(i+1))
        else:
            exact = 0
        numerical = midpoint_rule(
            f=lambda x: sin((i+1)*pi*x/2))
        print((i, abs(exact - numerical)))
```

The output now is around $10^{-10}$ and we take that as a sign that
our exact results are reliable.

**Carefully check symbolic computations!**

The example above shows how `sympy` can fail.
[Wolfram Alpha](http://wolframalpha.com) does a better job: writing
`integrate sin(k*x*pi/2) from 0 to 1` (use `k` instead of `i` since the latter
is the imaginary unit) returns the [result](http://www.wolframalpha.com/input/?i=integrate+sin%28k*x*pi%2F2%29+from+0+to+1) $4\sin^2(\pi k/4)/(\pi k)$,
which coincides with out result.

There are three general techniques
to verify a symbolic computation:

 * Use alternative software like Wolfram Alpha for comparison

 * Check that the result satisfies the problem to be solved

 * Make a high-resolution numerical approximation and compare

(The second technique is not so applicable here, since we work with
a definite integral, but one could compute the indefinite integral
instead, which is done correctly by `sympy`, and discuss values for
$x=1$.)

The final result for $c_i$ is now

$$
c_i = \frac{b_i}{A_{i,i}} = \left\lbrace\begin{array}{ll}
-\frac{16}{\pi^3(i+1)^3}, & i\hbox{ even, or } i \hbox{ mod } 2 = 0\\
-\frac{32}{\pi^3(i+1)^3}, & (i-1)\hbox{ mod } 4 = 0,\\
0, & (i+1)\hbox{ mod } 4 = 0
\end{array}\right.
$$

We recognize that for $i$ even, say $i=2k$ for integer $k$, we
have exactly the same result as in b):

$$
-\sum_k \frac{16}{\pi^3(2k+1)^3}\sin((2k+1)x\frac{\pi x}{2}),
$$

but we get an additional set of terms for $i=4k+1$,

$$
\begin{equation}
-\sum_k \frac{32}{\pi^3(i+1)^3}\sin((4k+1)x\frac{\pi x}{2}){\thinspace .}
\label{fem:deq:exer:tension:cable:badterms} \tag{63}
\end{equation}
$$

We can modify the software from c) to compute the approximate $u$
with the present set of basis functions and coefficients:

```python
def sine_sum_d(x, N):
    s = 0
    from numpy import pi, sin, zeros
    u = []  # u[k] is the sum i=0,...,k
    k = 0
    for i in range(N+1):
        if i % 2 == 0:       # even i
            s +=   - 16.0/((i+1)**3*pi**3)*sin((i+1)*pi*x/2)
        elif (i-1) % 4 == 0:   # 1, 5, 9, 13, 17
            s += - 2*16.0/((i+1)**3*pi**3)*sin((i+1)*pi*x/2)
        else:
            s += 0
        u.append(s.copy())
    return u

def plot_sine_sum_d():
    from numpy import linspace
    x = linspace(0, 1, 501)  # coordinates for plot
    u = sine_sum_d(x, N=20)
    u_e = 0.5*x*(x-2)
    N_values = 0, 1, 2, 3, 20
    for k in N_values:
        plt.plot(x, u[k])
    plt.plot(x, u_e)
    plt.legend(['N=%d' % k for k in N_values] + ['exact'],
               loc='upper right')
    plt.xlabel('$x$');  plt.ylabel('$u$')
    #plt.axis([0.9, 1, -0.52, -0.49])
    plt.savefig('tmpd.png'); plt.savefig('tmpd.pdf')
```

The approximations for $N=0,1,3,20$ appear below.

While the approximation for $N=0$ coincides with the one in b), we
see that $N=1$ and higher values of $N$ lead to a clearly wrong curve.
This strange feature has to be investigated!

Let us start by plotting the basis functions for $i=0,1,\ldots,7$:

We observe from the figure that all the basis functions corresponding to
even $i$ are symmetric around $x=1$, which is an important property of
the solution. The functions for odd $i$ are anti-symmetric. However,
for $i=3,7,11,\ldots$ the basis function has an integer number of
periods on $[0,1]$ so the integral becomes zero, $c_i=0$, and
consequently
there is no effect from these functions. The functions corresponding
to $i=1,5,9,13,\ldots$ are anti-symmetric around $x=1$ with nonzero
coefficients. The derivative of an anti-symmetric function at the point
of anti-symmetry is unity in size. Since the derivatives of all the
basis functions corresponding to even $i$ vanish at $x=1$, the
extra terms ($i=1,5,9,13,\ldots$) in ([63](#fem:deq:exer:tension:cable:badterms))
have a nonzero derivative, resulting in $u'(1)\neq 0$. That is,
these terms destroy the solution!

But we computed $c_i$ by a Galerkin method, which is equivalent to a
least squares method, which gives us the "best" approximation possible?
That is true, but it is the best approximation in the chosen space $V$.
The problem is that we have populated (or rather polluted) the space
$V$ with some basis functions that have a wrong mathematical property: they
are anti-symmetric around $x=1$.

**e)**
Now we drop the symmetry condition at $x=1$ and extend the domain to
$[0,2]$ such that it covers the entire (scaled) physical cable. The
problem now reads

$$
u'' = 1,\quad x\in (0,2),\quad u(0)=u(2)=0{\thinspace .}
$$

This time we need basis functions that are zero at $x=0$ and $x=2$.
The set $\sin((i+1)\frac{\pi x}{2})$ from d) is a candidate since
they vanish $x=2$ for any $i$. Compute the approximation in this case.
Why is this approximation without the problem that this set of
basis functions introduced in d)?

**Solution.**
The formulas are almost the same as in d), only the integration domain
is different. Since the sine functions or orthogonal on $[0,1]$, they
are also orthogonal on $[0,2]$. Because

```python
integrate(cos((i+1)*pi*x/2)**2, (x, 0, 2))
```

we get (in Galerkin's method)

$$
\begin{align*}
A_{i,i} &= (i+1)(j+1)\frac{\pi^2}{4}\int_0^2 \cos((i+1)\frac{\pi x}{2})
\cos((i+1)\frac{\pi x}{2})dx\\
& = (i+1)^2\frac{\pi^2}{4}{\thinspace .}
\end{align*}
$$

and

$$
b_i = -\int_0^2 \sin((i+1)\frac{\pi x}{2})dx = \frac{2}{\pi (i+1)}(\cos((i+1)\pi) - 1){\thinspace .}
$$

We have that $\cos((i+1)\pi = -1$ for $i$ even and
$\cos((i+1)\pi = 1$ for $i$ odd. That is,

$$
b_i =\left\lbrace\begin{array}{ll}
-\frac{4}{\pi (i+1)}, & i\hbox{ even }\\
0, & i\hbox{ odd }
\end{array}\right.
$$

The coefficients become

$$
c_i =\frac{b_i}{A_{i,i}} =\left\lbrace\begin{array}{ll}
-\frac{16}{\pi^3(i+1)^3}, & i\hbox{ even }\\
0, & i\hbox{ odd }
\end{array}\right.
$$

Introducing $i=2k$ and then switching from $k$ to $i$ as summation index
gives $c_i = -\frac{16}{\pi^3(2i+1)^3}$ and

$$
u(x) = -\sum_{i=0}^N \frac{16}{\pi^3(2i+1)^3}\sin((i+1)\frac{\pi x}{2}),
$$

which is the same expansion as in b).

The reason why the basis functions ${\psi}_i=\sin((i+1)\frac{\pi x}{2})$
work well in this case is that the problematic functions for $i=1,5,\ldots$
in d) now live on $[0,2]$ instead of $[0,1]$. On $[0,2]$ these functions
have an integer number of periods such that the integral from 0 to 2
becomes zero. These basis functions are therefore excluded from the
expansion since their coefficients vanish.
The lesson learned is that two equivalent boundary value
problems may make different demands to the basis functions.

Filename: `cable_sin`.

## Exercise 3: Compute the deflection of a cable with power functions

**a)**
Repeat [Exercise 2: Compute the deflection of a cable with sine functions](#fem:deq:exer:tension:cable) b), but work with
the space

$$
V = \hbox{span}\{x, x^2, x^3, x^4, \ldots\}{\thinspace .}
$$

Choose the dimension of $V$ to be 4 and observe that the exact solution
is recovered by the Galerkin method.

**Hint.**
Use the `solver` function from `varform1D.py`.

**Solution.**
The Galerkin formulation of $u''=1$, $u(0)=0$, $u'(1)=0$, reads

$$
(u',v') = -(1,v)\quad\forall v\in V,
$$

and the linear system becomes

$$
\sum_{j=}^N ({\psi}_i', {\psi}_j')c_j = -(1,{\psi}_i),\quad i=0,1,\ldots,N{\thinspace .}
$$

The `varform1D.solver` function needs a function specifying the integrands
on the left- and right-hand sides of the variational formulation.
Moreover, we must compute a dictionary of ${\psi}_i$ and ${\psi}_i'$.
The appropriate code becomes

```python
from varform1D import solver
import sympy as sym
x, b = sym.symbols('x b')
f = 1

# Compute basis functions and their derivatives
N = 4
psi = {0: [x**(i+1) for i in range(N+1)]}
psi[1] = [sym.diff(psi_i, x) for psi_i in psi[0]]

# Galerkin

def integrand_lhs(psi, i, j):
    return psi[1][i]*psi[1][j]

def integrand_rhs(psi, i):
    return -f*psi[0][i]

Omega = [0, 1]

u, c = solver(integrand_lhs, integrand_rhs, psi, Omega,
              verbose=True, symbolic=True)
print(('Galerkin solution u:', sym.simplify(sym.expand(u))))
```

Running this code gives the output

        solution u: x*(x - 2)/2

which coincides with the exact solution ($c_3=c_4=0$).

**b)**
What happens if we use a least squares method for this problem with
the basis in a)?

**Solution.**
The least squares formulation leads to

$$
(R,\frac{\partial R}{\partial c_i}=0,\quad i=0,\ldots,N,
$$

with

$$
R = 1 - u'' = 1 - \sum_jc_j{\psi}_j''{\thinspace .}
$$

We have

$$
\frac{\partial R}{\partial c_i} = {\psi}_i'',
$$

leading to the equations

$$
(1 + \sum_jc_j{\psi}_j'', {\psi}_i''),\quad i=0,\ldots,N,
$$

which is a linear system

$$
\sum_{j=0}^N({\psi}_j'',{\psi}_i'') = (-1,{\psi}_i''),\quad i=0,\ldots,N{\thinspace .}
$$

The fundamental problem with the basis in a) is that ${\psi}_0''=0$, so
if power functions of $x$ are wanted, we need to work with the basis
$V=\hbox{span}\{x^2, x^3,\ldots\}$. If we do so, we can easily modify
the code from a),

```python
# Least squares
psi = {0: [x**(i+2) for i in range(N+1)]}
psi[1] = [sym.diff(psi_i, x) for psi_i in psi[0]]
psi[2] = [sym.diff(psi_i, x) for psi_i in psi[1]]

def integrand_lhs(psi, i, j):
    return psi[2][i]*psi[2][j]

def integrand_rhs(psi, i):
    return -f*psi[2][i]

Omega = [0, 1]

u, c = solver(integrand_lhs, integrand_rhs, psi, Omega,
              verbose=True, symbolic=True)
print(('solution u:', sym.simplify(sym.expand(u))))
```

The result is $u=-\frac{1}{2} x^2$. This function does not obey $u'(1)=0$ and
is completely wrong. In this least squares method we cannot access the basis
function $x$, which is needed in the exact solution, and we have no means
to obtain $u'(1)=0$.

**Remark.** There is a modification of the least squares method that
can be applied here. The
problem $u''=1$ must be rewritten as a system of two equations,
$u_1'=u_2$, $u_2' =1$. We expand $u_1=\sum_{j=0}^N c_j{\psi}_j$ and
$u_2=\sum_{j=0}^N d_j{\psi}_j$. The residuals in both equations are
added, squared, and differentiated with respect to $c_i$ and $d_i$,
$i=0,\ldots,N$. The result is a coupled equation system for the
$c_i$ and $d_i$ coefficients.

Filename: `cable_xn`.

## Exercise 4: Check integration by parts

Consider the Galerkin method for the problem involving $u$
in [Exercise 2: Compute the deflection of a cable with sine functions](#fem:deq:exer:tension:cable).
Show that the formulas for $c_j$ are independent of whether we perform
integration by parts or not.

**Solution.**
The Galerkin method is

$$
(u'',v)=(1,v)\quad\forall v\in V,
$$

and with the choice of $V$ we get

$$
\begin{align*}
A_{i,j} &=-(i+1)^2\pi^2 \int_0^1\sin^2((i+1)\frac{\pi x}{2})dx,\\
b_i &= \int_0^1\sin((i+1)\frac{\pi x}{2})dx
\end{align*}
$$

From [Exercise 2: Compute the deflection of a cable with sine functions](#fem:deq:exer:tension:cable) we realize that
the integrals are the same as in the least squares method, and
those results were identical to those of the Galerkin method with
integration by parts.

Filename: `cable_integr_by_parts`.