# A Review of Welford's Paper

In 1962, B.P. Welford published a paper titled [Note on a Method for Calculating Corrected Sums of Squares and Products](http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.302.7503&rep=rep1&type=pdf). Let's review some important identities/derivations from this paper as they will come in handy later. Note that we use zero-based indexing when referencing elements of our time series.

### Deriving Equation (1)

\begin{align}
    \mu_{Q_{i,n}} ={}&
        \sum \limits _{0 \leq j \lt n} \frac{q_{i+j}}{n}
    \\
    ={}&
        \frac{q_{i}}{n} + \frac{q_{i+1}}{n} + \ldots + \frac{q_{i+n-2}}{n} + \frac{q_{i+n-1}}{n}
    \\
    ={}&
        1 \cdot \frac{q_{i}}{n} + 1 \cdot \frac{q_{i+1}}{n} + \ldots + 1 \cdot \frac{q_{i+n-2}}{n} + \frac{q_{i+n-1}}{n}
    \\
    ={}&
        \frac{n-1}{n-1} \frac{q_{i}}{n} + \frac{n-1}{n-1} \frac{q_{i+1}}{n} + \ldots + \frac{n-1}{n-1} \frac{q_{i+n-2}}{n} + \frac{q_{i+n-1}}{n}
    \\
    ={}&
        \frac{n-1}{n} \frac{q_{i}}{n-1} + \frac{n-1}{n} \frac{q_{i+1}}{n-1} + \ldots + \frac{n-1}{n} \frac{q_{i+n-2}}{n} + \frac{q_{i+n-1}}{n}
    \\
    ={}&
        \frac{n-1}{n}
        \left(
            \frac{q_{i}}{n-1} + \frac{q_{i+1}}{n-1} + \ldots + \frac{q_{i+n-2}}{n-1}
        \right)
        + \frac{q_{i+n-1}}{n}
    \\
    ={}&
        \frac{n-1}{n}
        \sum \limits _{0 \leq j \lt n-1} \frac{q_{i+j}}{n-1}
        + \frac{q_{i+n-1}}{n}
    \\
    ={}&
        \frac{n-1}{n}
        \mu_{Q_{i,n-1}}
        + \frac{q_{i+n-1}}{n}
    \\
    \mu_{Q_{i,m}} ={}&
        \frac{m-1}{m}
        \mu_{Q_{i,m-1}}
        + \frac{q_{i+m-1}}{m}
    \\
\end{align}

### Deriving Equation (2)

\begin{align}
    j < n-1, \;\;\;\;\;
    q_{i+j} - \mu_{Q_{i,n}} ={}&
        q_{i+j} - \frac{n-1}{n}
        \mu_{Q_{i,n-1}}
        - \frac{q_{i+n-1}}{n}
    \\
    ={}&
        q_{i+j}
        - \frac{n}{n} \mu_{Q_{i,n-1}}
        + \frac{\mu_{Q_{i,n-1}}}{n}
        - \frac{q_{i+n-1}}{n}
    \\
    ={}&
        q_{i+j}
        - \mu_{Q_{i,n-1}}
        + \frac{1}{n}
        \left(
        -q_{i+n-1} + \mu_{Q_{i,n-1}}
        \right)
    \\
    ={}&
        q_{i+j}
        - \mu_{Q_{i,n-1}}
        - \frac{1}{n}
        \left(
        q_{i+n-1} - \mu_{Q_{i,n-1}}
        \right)
    \\
    q_{i+j} - \mu_{Q_{i,m}} ={}&
        q_{i+j}
        - \mu_{Q_{i,m-1}}
        - \frac{1}{m}
        \left(
        q_{i+m-1} - \mu_{Q_{i,m-1}}
        \right)
    \\
\end{align}

### Deriving Equation (3)

\begin{align}
    j = n-1, \;\;\;\;\;
    q_{i+n-1} - \mu_{Q_{i,n}} &=
        q_{i+n-1} - \frac{n-1}{n}
        \mu_{Q_{i,n-1}}
        - \frac{q_{i+n-1}}{n}
    \\
    &=
        - \frac{n-1}{n}
        \mu_{Q_{i,n-1}}
        - \frac{q_{i+n-1}}{n} + q_{i+n-1}
    \\
    &=
        - \frac{n-1}{n}
        \mu_{Q_{i,n-1}}
        + q_{i+n-1}
        \left(
        -\frac{1}{n} + 1
        \right)
    \\
    &=
        - \frac{n-1}{n}
        \mu_{Q_{i,n-1}}
        + q_{i+n-1}
        \left(
        -\frac{1}{n} + \frac{n}{n}
        \right)
    \\
    &=
        - \frac{n-1}{n}
        \mu_{Q_{i,n-1}}
        + q_{i+n-1}
        \left(
        -\frac{1}{n} + \frac{n}{n}
        \right)
    \\
    &=
        - \frac{n-1}{n}
        \mu_{Q_{i,n-1}}
        + q_{i+n-1}
        \left(
        \frac{n-1}{n}
        \right)
    \\
    &=
        \frac{n-1}{n}
        \left(
            q_{i+n-1}
            -
            \mu_{Q_{i,n-1}}
        \right)
    \\
    q_{i+m-1} - \mu_{Q_{i,m}} &= \frac{m-1}{m}
        \left(
            q_{i+m-1}
            -
            \mu_{Q_{i,m-1}}
        \right)
\end{align}

### Deriving Equation (I) - The Corrected Sum of Squares

for $n = 1, 2, \ldots, m$

\begin{align}
    S(Q_{i,n}) ={}&
        \sum \limits _{0 \leq j \lt n}
            \left(
                q_{i+j} - \mu_{Q_{i,n}}
            \right) ^2
    \\
    ={}&
        \sum \limits _{0 \leq j \lt n-1}
                (q_{i+j} - \mu_{Q_{i,n}})^2
        +
        \left(
            q_{i+n-1} - \mu_{Q_{i,n}}
        \right) ^2
    \\
    ={}&
        \sum \limits _{0 \leq j \lt n-1}
            \left[
                q_{i+j}
                - \mu_{Q_{i,n-1}}
                - \frac{1}{n}
                \left(
                    q_{i+n-1} - \mu_{Q_{i,n-1}}
                \right)
            \right] ^2
        +
        \left[
            \frac{n-1}{n}
            \left(
                q_{i+n-1}
                -
                \mu_{Q_{i,n-1}}
            \right)
        \right] ^2
    \\
    ={}&
        \sum \limits _{0 \leq j \lt m-1}
            \left[
                \left(
                    q_{i+j}
                    - \mu_{Q_{i,n-1}}
                \right)
                - \frac{1}{n}
                \left(
                    q_{i+n-1} - \mu_{Q_{i,n-1}}
                \right)
            \right] ^2
        +
        \left(
            \frac{n-1}{n}
        \right) ^2
        \left(
            q_{i+n-1}
            -
            \mu_{Q_{i,n-1}}
        \right) ^2
    \\
    ={}&
        \sum \limits _{0 \leq j \lt n-1}
            \left[
                \left(
                    q_{i+j}
                    - \mu_{Q_{i,n-1}}
                \right) ^2
                - \frac{2}{n}
                \left(
                    q_{i+j}
                    - \mu_{Q_{i,n-1}}
                \right)
                \left(
                    q_{i+n-1} - \mu_{Q_{i,n-1}}
                \right)
                + \frac{1}{n^2}
                \left(
                    q_{i+n-1} - \mu_{Q_{i,n-1}}
                \right) ^2
            \right]
        +
        \frac{(n-1)^2}{n^2}
        \left(
            q_{i+n-1}
            -
            \mu_{Q_{i,n-1}}
        \right) ^2
    \\
        ={}&
        \sum \limits _{0 \leq j \lt n-1}
            \left(
                q_{i+j}
                - \mu_{Q_{i,n-1}}
            \right) ^2
        -
        \sum \limits _{0 \leq j \lt n-1}
            \frac{2}{n}
            \left(
                q_{i+j}
                - \mu_{Q_{i,n-1}}
            \right)
            \left(
                q_{i+n-1} - \mu_{Q_{i,n-1}}
            \right)
        +
        \sum \limits _{0 \leq j \lt n-1}
            \frac{1}{n^2}
            \left(
                q_{i+n-1} - \mu_{Q_{i,n-1}}
            \right) ^2
        +
        \frac{(n-1)^2}{n^2}
        \left(
            q_{i+n-1}
            -
            \mu_{Q_{i,n-1}}
        \right) ^2
    \\
    ={}&
        \sum \limits _{0 \leq j \lt n-1}
            \left(
                q_{i+j}
                - \mu_{Q_{i,n-1}}
            \right) ^2
        -
        2 \left(
            q_{i+n-1} - \mu_{Q_{i,n-1}}
          \right)
        \frac{1}{n}
        \sum \limits _{0 \leq j \lt n-1}
                \left(
                    q_{i+j}
                    - \mu_{Q_{i,n-1}}
                \right)
        +
        \sum \limits _{0 \leq j \lt n-1}
            \frac{1}{n^2}
            \left(
                q_{i+n-1} - \mu_{Q_{i,n-1}}
            \right) ^2
        +
        \frac{(n-1)^2}{n^2}
        \left(
            q_{i+n-1}
            -
            \mu_{Q_{i,n-1}}
        \right) ^2
    \\
        ={}&
        \sum \limits _{0 \leq j \lt n-1}
            \left(
                q_{i+j}
                - \mu_{Q_{i,n-1}}
            \right) ^2
        -
        2 \left(
            q_{i+n-1} - \mu_{Q_{i,n-1}}
          \right)
        \cdot 0
        +
        \sum \limits _{0 \leq j \lt n-1}
            \frac{1}{n^2}
            \left(
                q_{i+n-1} - \mu_{Q_{i,n-1}}
            \right) ^2
        +
        \frac{(n-1)^2}{n^2}
        \left(
            q_{i+n-1}
            -
            \mu_{Q_{i,n-1}}
        \right) ^2
    \\
    ={}&
        \sum \limits _{0 \leq j \lt n-1}
            \left(
                q_{i+j}
                - \mu_{Q_{i,n-1}}
            \right) ^2
        +
        \frac{n-1}{n^2}
        \left(
            q_{i+n-1} - \mu_{Q_{i,n-1}}
        \right) ^2
        +
        \frac{(n-1)^2}{n^2}
        \left(
            q_{i+n-1}
            -
            \mu_{Q_{i,n-1}}
        \right) ^2
    \\
    ={}&
        \sum \limits _{0 \leq j \lt n-1}
            \left(
                q_{i+j}
                - \mu_{Q_{i,n-1}}
            \right) ^2
        +
        \left[
            \frac{n-1}{n^2}
            +
            \frac{(n-1)^2}{n^2}
        \right]
        \left(
            q_{i+n-1}
            -
            \mu_{Q_{i,n-1}}
        \right) ^2
    \\
    ={}&
        \sum \limits _{0 \leq j \lt n-1}
            \left(
                q_{i+j}
                - \mu_{Q_{i,n-1}}
            \right) ^2
        +
        \left[
            \frac{n-1+n^2-2n+1}{n^2}
        \right]
        \left(
            q_{i+n-1}
            -
            \mu_{Q_{i,n-1}}
        \right) ^2
    \\
    ={}&
        \sum \limits _{0 \leq j \lt n-1}
            \left(
                q_{i+j}
                - \mu_{Q_{i,n-1}}
            \right) ^2
        +
        \left[
            \frac{n^2-n}{n^2}
        \right]
        \left(
            q_{i+n-1}
            -
            \mu_{Q_{i,n-1}}
        \right) ^2
    \\
    ={}&
        S(Q_{i,n-1})
        +
        \left(
            \frac{n-1}{n}
        \right)
        \left(
            q_{i+n-1}
            -
            \mu_{Q_{i,n-1}}
        \right) ^2
    \\
    S(Q_{i,m}) ={}&
        S({Q_{i,m-1}})
        +
        \left(
            \frac{m-1}{m}
        \right)
        \left(
            q_{i+m-1}
            -
            \mu_{Q_{i,m-1}}
        \right) ^2
    \\
\end{align}

### Deriving Equation (II) - The Corrected Sum of Products

\begin{align}
    S(Q_{i,n}, T_{i,n}) ={}&
        \sum \limits _{0 \leq j \lt n}
            \left(
                q_{i+j} - \mu_{Q_{i,n}}
            \right)
            \left(
                t_{i+j} - M_{T_{i,n}}
            \right)
    \\
    ={}&
       \sum \limits _{0 \leq j \lt n-1}
            \left(
                q_{i+j} - \mu_{Q_{i,n}}
            \right)
            \left(
                t_{i+j} - M_{T_{i,n}}
            \right)
        +
        \left(
            q_{i+n-1} - \mu_{Q_{i,n}}
        \right)
        \left(
            t_{i+n-1} - M_{T_{i,n}}
        \right)
    \\
    ={}&
       \sum \limits _{0 \leq j \lt n-1}
            \left[
                q_{i+j}
                - \mu_{Q_{i,n-1}}
                - \frac{1}{n}
                \left(
                    q_{i+n-1} - \mu_{Q_{i,n-1}}
                \right)
            \right]
            \left[
                t_{i+j}
                - M_{T_{i,n-1}}
                - \frac{1}{n}
                \left(
                    t_{i+n-1} - M_{T_{i,n-1}}
                \right)
            \right]
        +
        \left[
            \frac{n-1}{n}
            \left(
                q_{i+n-1}
                -
                \mu_{Q_{i+n-1}}
            \right)
        \right]
        \left[
            \frac{n-1}{n}
            \left(
                t_{i+n-1}
                -
                M_{T_{i,n-1}}
            \right)
        \right]
    \\
    ={}&
        \sum \limits _{0 \leq j \lt n-1}
            \left[
                \left(
                    q_{i+j}
                    - \mu_{Q_{i,n-1}}
                \right)
                - \frac{1}{n}
                \left(
                    q_{i+n-1} - \mu_{Q_{i,n-1}}
                \right)
            \right]
            \left[
                \left(
                    t_{i+j}
                    - M_{T_{i,n-1}}
                \right)
                - \frac{1}{n}
                \left(
                    t_{i+n-1} - M_{T_{i,n-1}}
                \right)
            \right]
        +
        \left(
            \frac{n-1}{n}
        \right)^2
        \left(
            q_{i+n-1}
            -
            \mu_{Q_{i,n-1}}
        \right)
        \left(
            t_{i+n-1}
            -
            M_{T_{i,n-1}}
        \right)
    \\
    ={}&
        \sum \limits _{0 \leq j \lt n-1}
            \left[
                \left(
                    q_{i+j}
                    - \mu_{Q_{i,n-1}}
                \right)
                \left(
                    t_{i+j}
                    - M_{T_{i,n-1}}
                \right)
                -
                \left(
                    q_{i+j}
                    - \mu_{Q_{i,n-1}}
                \right)
                \frac{1}{n}
                \left(
                    t_{i+n-1} - M_{T_{i,n-1}}
                \right)
                - \frac{1}{n}
                \left(
                    q_{i+n-1} - \mu_{Q_{i,n-1}}
                \right)
                \left(
                    t_{i+j}
                    - M_{T_{i,n-1}}
                \right)
                +
                \frac{1}{n}
                \left(
                    q_{i+n-1} - \mu_{Q_{i,n-1}}
                \right)
                \frac{1}{n}
                \left(
                    t_{i+n-1} - M_{T_{i,n-1}}
                \right)
            \right]
        \\
        {}&
        +
        \left(
            \frac{n-1}{n}
        \right)^2
        \left(
            q_{i+n-1}
            -
            \mu_{Q_{i,n-1}}
        \right)
        \left(
            t_{i+n-1}
            -
            M_{T_{i,n-1}}
        \right)
    \\
    ={}&
        \sum \limits _{0 \leq j \lt n-1}
            \left(
                q_{i+j}
                - \mu_{Q_{i,n-1}}
            \right)
            \left(
                t_{i+j}
                - M_{T_{i,n-1}}
            \right)
        -\frac{1}{n}
        \left(
            t_{i+n-1} - M_{T_{i,n-1}}
        \right)
        \sum \limits _{0 \leq j \lt n-1}
            \left(
                q_{i+j}
                - \mu_{Q_{i,n-1}}
            \right)
        - \frac{1}{n}
        \left(
            q_{i+n-1} - \mu_{Q_{i,n-1}}
        \right)
        \sum \limits _{0 \leq j \lt n-1}
            \left(
                t_{i+j}
                - M_{T_{i,n-1}}
            \right)
        +\frac{1}{n^2}
        \sum \limits _{0 \leq j \lt n-1}
            \left(
                q_{i+n-1} - \mu_{Q_{i,n-1}}
            \right)
            \left(
                t_{i+n-1} - M_{T_{i,n-1}}
            \right)
        \\
        {}&
        +
        \left(
            \frac{n-1}{n}
        \right)^2
        \left(
            q_{i+n-1}
            -
            \mu_{Q_{i,n-1}}
        \right)
        \left(
            t_{i+n-1} - M_{T_{i,n-1}}
        \right)
    \\
    ={}&
        \sum \limits _{0 \leq j \lt n-1}
            \left(
                q_{i+j}
                - \mu_{Q_{i,n-1}}
            \right)
            \left(
                t_{i+j}
                - M_{T_{i,n-1}}
            \right)
        -\frac{1}{n}
        \left(
            t_{i+n-1} - M_{T_{i,n-1}}
        \right)
        \sum \limits _{0 \leq j \lt n-1}
            \left(
                q_{i+j}
                - \mu_{Q_{i,n-1}}
            \right)
        - \frac{1}{n}
        \left(
            q_{i+n-1} - \mu_{Q_{i,n-1}}
        \right)
        \sum \limits _{0 \leq j \lt n-1}
            \left(
                t_{i+j}
                - M_{T_{i,n-1}}
            \right)
        +\frac{n-1}{n^2}
        \left(
            q_{i+n-1} - \mu_{Q_{i,n-1}}
        \right)
        \left(
            t_{i+n-1} - M_{T_{i,n-1}}
        \right)
        \\
        {}&
        +
        \left(
            \frac{n-1}{n}
        \right)^2
        \left(
            q_{i+n-1}
            -
            \mu_{Q_{i,n-1}}
        \right)
        \left(
            t_{i+n-1}
            -
            M_{T_{i,n-1}}
        \right)
    \\
    ={}&
        S(Q_{i,n-1}, T_{i,n-1})
        -\frac{1}{n}
        \left(
            t_{i+n-1} - M_{T_{i,n-1}}
        \right)
        \sum \limits _{0 \leq j \lt n-1}
            \left(
                q_{i+j}
                - \mu_{Q_{i,n-1}}
            \right)
        - \frac{1}{n}
        \left(
            q_{i+n-1} - \mu_{Q_{i,n-1}}
        \right)
        \sum \limits _{0 \leq j \lt n-1}
            \left(
                t_{i+j}
                - M_{T_{i,n-1}}
            \right)
        +\frac{n-1}{n^2}
        \left(
            q_{i+n-1} - \mu_{Q_{i,n-1}}
        \right)
        \left(
            t_{i+n-1} - M_{T_{i,n-1}}
        \right)
        \\
        {}&
        +
        \left(
            \frac{n-1}{n}
        \right)^2
        \left(
            q_{i+n-1}
            -
            \mu_{Q_{i,n-1}}
        \right)
        \left(
            t_{i+n-1}
            -
            M_{T_{i,n-1}}
        \right)
    \\
    ={}&
        S(Q_{i,n-1}, T_{i,n-1})
        -\frac{1}{n}
        \left(
            t_{i+n-1} - M_{T_{i,n-1}}
        \right)
        \cdot 0
        - \frac{1}{n}
        \left(
            q_{i+n-1} - \mu_{Q_{i,n-1}}
        \right)
        \cdot 0
        +\frac{n-1}{n^2}
        \left(
            q_{i+n-1} - \mu_{Q_{i,n-1}}
        \right)
        \left(
            t_{i+n-1} - M_{T_{i,n-1}}
        \right)
        +
        \left(
            \frac{n-1}{n}
        \right)^2
        \left(
            q_{i+n-1}
            -
            \mu_{Q_{i,n-1}}
        \right)
        \left(
            t_{i+n-1}
            -
            M_{T_{i,n-1}}
        \right)
    \\
    ={}&
        S(Q_{i,n-1}, T_{i,n-1})
        +\frac{n-1}{n^2}
        \left(
            q_{i+n-1} - \mu_{Q_{i,n-1}}
        \right)
        \left(
            t_{i+n-1} - M_{T_{i,n-1}}
        \right)
        +
        \left(
            \frac{n-1}{n}
        \right)^2
        \left(
            q_{i+n-1}
            -
            \mu_{Q_{i,n-1}}
        \right)
        \left(
            t_{i+n-1}
            -
            M_{T_{i,n-1}}
        \right)
    \\
    ={}&
        S(Q_{i,n-1}, T_{i,n-1})
        +
        \left[
            \frac{n-1}{n^2}
            +
            \left(
                \frac{n-1}{n}
            \right)^2
        \right]
        \left(
            q_{i+n-1} - \mu_{Q_{i,n-1}}
        \right)
        \left(
            t_{i+n-1} - M_{T_{i,n-1}}
        \right)
    \\
    ={}&
        S(Q_{i,n-1}, T_{i,n-1})
        +
        \left[
            \frac{n-1}{n^2}
            +
            \frac{n^2-2n+1}{n^2}
        \right]
        \left(
            q_{i+n-1} - \mu_{Q_{i,n-1}}
        \right)
        \left(
            t_{i+n-1} - M_{T_{i,n-1}}
        \right)
    \\
    ={}&
        S(Q_{i+n-1}, T_{i,n-1})
        +
        \left(
            \frac{n-1+n^2-2n+1}{n^2}
        \right)
        \left(
            q_{i+n-1} - \mu_{Q_{i,n-1}}
        \right)
        \left(
            t_{i+n-1} - M_{T_{i,n-1}}
        \right)
    \\
    ={}&
        S(Q_{i,n-1}, T_{i,n-1})
        +
        \left(
            \frac{n^2-n}{n^2}
        \right)
        \left(
            q_{i+n-1} - \mu_{Q_{i,n-1}}
        \right)
        \left(
            t_{i+n-1} - M_{T_{i,n-1}}
        \right)
    \\
    ={}&
        S(Q_{i,n-1}, T_{i,n-1})
        +
        \left(
            \frac{n-1}{n}
        \right)
        \left(
            q_{i+n-1} - \mu_{Q_{i,n-1}}
        \right)
        \left(
            t_{i+n-1} - M_{T_{i,n-1}}
        \right)
    \\
\end{align}

\begin{align}
    S(Q_{i,m}, T_{i,m}) ={}&
        S(Q_{i,m-1}, T_{i,m-1})
        +
        \left(
            \frac{m-1}{m}
        \right)
        \left(
            q_{i+m-1} - \mu_{Q_{i,m-1}}
        \right)
        \left(
            t_{i+m-1} - M_{T_{i,m-1}}
        \right)
    \\
    S(Q_{i-1,m}, T_{i-1,m}) ={}&
        S(Q_{i-1,m-1}, T_{i-1,m-1})
        +
        \left(
            \frac{m-1}{m}
        \right)
        \left(
            q_{i+m-2} - \mu_{Q_{i-1,m-1}}
        \right)
        \left(
            t_{i+m-2} - M_{T_{i-1,m-1}}
        \right)
    \\
\end{align}

Building on Equation (II) and also the fact that:

\begin{align}
    \mu_{Q_{i-1,n}}
    ={}& \frac{n-1}{n} \mu_{Q_{i-1,n-1}} + \frac{q_{i+n-2}}{n}
    \\
    ={}&  \mu_{Q_{i-1,n-1}} - \frac{1}{n} \mu_{Q_{i-1,n-1}} + \frac{q_{i+n-2}}{n}
    \\
    ={}&  \mu_{Q_{i-1,n-1}} + \frac{q_{i+n-2}}{n} - \frac{\mu_{Q_{i-1,n-1}}}{n}
    \\
    ={}& \frac{1}{n} \sum \limits _{0 \leq j \lt n-1}
        q_{i+j-1}
        + \frac{q_{i+n-2}}{n}
        - \frac{\mu_{Q_{i-1,n-1}}}{n}
    \\
    ={}&
        \frac{q_{i-1}}{n}
        +
        \frac{1}{n} \sum \limits _{1 \leq j \lt n-1}
            q_{i+j-1}
        + \frac{q_{i+n-2}}{n}
        - \frac{\mu_{Q_{i-1,n-1}}}{n}
    \\
    ={}& \frac{q_{i-1}}{n}
        + \frac{1}{n} \sum \limits _{0 \leq j \lt n-1} q_{i+j}
        - \frac{\mu_{Q_{i-1,n-1}}}{n}
    \\
    ={}& \frac{q_{i-1}}{n} + \mu_{Q_{i,n-1}} - \frac{\mu_{Q_{i-1,n-1}}}{n}
    \\
    ={}& \mu_{Q_{i,n-1}} + \frac{1}{n} \left( q_{i-1} - \mu_{Q_{i-1,n-1}} \right)
    \\
\end{align}

we can actually express $S(Q_{i-1,n},T_{i-1,n})$ with respect to $S(Q_{i,n-1},T_{i,n-1}) = \sum \limits _{0 \leq j \lt n-1} \left( q_{i+j} - \mu_{Q_{i,n-1}} \right) \left( t_{i+j} - M_{T_{i,n-1}} \right)$ and $\left( q_{i-1} - \mu_{Q_{i,n-1}} \right) \left( t_{i-1} - M_{T_{i,n-1}} \right)$:

\begin{align}
    S(Q_{i-1,n}, T_{i-1,n}) ={}&
        \sum \limits _{0 \leq j \lt n}
            \left(
                q_{i+j-1} - \mu_{Q_{i-1,n}}
            \right)
            \left(
                t_{i+j-1} - M_{T_{i-1,n}}
            \right)
    \\
    ={}&
        \left(
            q_{i-1} - \mu_{Q_{i-1,n}}
        \right)
        \left(
            t_{i-1} - M_{T_{i-1,n}}
        \right)
        +
        \sum \limits _{1 \leq j \lt n}
            \left(
                q_{i+j-1} - \mu_{Q_{i-1,n}}
            \right)
            \left(
                t_{i+j-1} - M_{T_{i-1,n}}
            \right)
    \\
    ={}&
        \left(
            q_{i-1} - \mu_{Q_{i-1,n}}
        \right)
        \left(
            t_{i-1} - M_{T_{i-1,n}}
        \right)
        +
        \sum \limits _{0 \leq j \lt n-1}
            \left(
                q_{i+j} - \mu_{Q_{i-1,n}}
            \right)
            \left(
                t_{i+j} - M_{T_{i-1,n}}
            \right)
    \\
    ={}&
        \sum \limits _{0 \leq j \lt n-1}
            \left(
                q_{i+j} - \mu_{Q_{i-1,n}}
            \right)
            \left(
                t_{i+j} - M_{T_{i-1,n}}
            \right)
        +
        \left(
            q_{i-1} - \mu_{Q_{i-1,n}}
        \right)
        \left(
            t_{i-1} - M_{T_{i-1,n}}
        \right)
    \\
    ={}&
        \sum \limits _{0 \leq j \lt n-1}
            \left[
                \left(
                    q_{i+j}
                    -
                    \mu_{Q_{i,n-1}}
                \right)
                -
                \frac{1}{n}
                \left(
                    q_{i-1} - \mu_{Q_{i-1,n-1}}
                \right)
            \right]
            \left[
                \left(
                    t_{i+j}
                    -
                    M_{T_{i,n-1}}
                \right)
                -
                \frac{1}{n}
                \left(
                    t_{i-1} - M_{T_{i-1,n-1}}
                \right)
            \right]
        +
        \left[
            \left(
                q_{i-1}
                -
                \mu_{Q_{i,n-1}}
            \right)
            -
            \frac{1}{n}
            \left(
                q_{i-1} - \mu_{Q_{i-1,n-1}}
            \right)
        \right]
        \left[
            \left(
                t_{i-1}
                -
                M_{T_{i,n-1}}
            \right)
            -
            \frac{1}{n}
            \left(
                t_{i-1} - M_{T_{i-1,n-1}}
            \right)
        \right]
    \\
    ={}&
        \sum \limits _{0 \leq j \lt n-1}
            \left[
                \left(
                    q_{i+j}
                    -
                    \mu_{Q_{i,n-1}}
                \right)
                \left(
                    t_{i+j}
                    -
                    M_{T_{i,n-1}}
                \right)
                -
                \left(
                    q_{i+j}
                    -
                    \mu_{Q_{i,n-1}}
                \right)
                \frac{1}{n}
                \left(
                    t_{i-1} - M_{T_{i-1,n-1}}
                \right)
                -
                \frac{1}{n}
                \left(
                    q_{i-1} - \mu_{Q_{i-1,n-1}}
                \right)
                \left(
                    t_{i+j}
                    -
                    M_{T_{i,n-1}}
                \right)
                +
                \frac{1}{n}
                \left(
                    q_{i-1} - \mu_{Q_{i-1,n-1}}
                \right)               \frac{1}{n}
                \left(
                    t_{i-1} - M_{T_{i-1,n-1}}
                \right)
            \right]
        \\
        &+
        \left[
            \left(
                q_{i-1}
                -
                \mu_{Q_{i,n-1}}
            \right)
            -
            \frac{1}{n}
            \left(
                q_{i-1} - \mu_{Q_{i-1,n-1}}
            \right)
        \right]
        \left[
            \left(
                t_{i-1}
                -
                M_{T_{i,n-1}}
            \right)
            -
            \frac{1}{n}
            \left(
                t_{i-1} - M_{T_{i-1,n-1}}
            \right)
        \right]
    \\
    ={}&
        \sum \limits _{0 \leq j \lt n-1}
            \left(
                q_{i+j}
                -
                \mu_{Q_{i,n-1}}
            \right)
            \left(
                t_{i+j}
                -
                M_{T_{i,n-1}}
            \right)
        -
        \sum \limits _{0 \leq j \lt n-1}
            \left(
                q_{i+j}
                -
                \mu_{Q_{i,n-1}}
            \right)
            \frac{1}{n}
            \left(
                t_{i-1} - M_{T_{i-1,n-1}}
            \right)
        -
        \sum \limits _{0 \leq j \lt n-1}
            \frac{1}{n}
            \left(
                q_{i-1} - \mu_{Q_{i-1,n-1}}
            \right)
            \left(
                t_{i+j}
                -
                M_{T_{i,n-1}}
            \right)
        +
        \sum \limits _{0 \leq j \lt n-1}
            \frac{1}{n}
            \left(
                q_{i-1} - \mu_{Q_{i-1,n-1}}
            \right)               \frac{1}{n}
            \left(
                t_{i-1} - M_{T_{i-1,n-1}}
            \right)
        \\
        &+
        \left[
            \left(
                q_{i-1}
                -
                \mu_{Q_{i,n-1}}
            \right)
            -
            \frac{1}{n}
            \left(
                q_{i-1} - \mu_{Q_{i-1,n-1}}
            \right)
        \right]
        \left[
            \left(
                t_{i-1}
                -
                M_{T_{i,n-1}}
            \right)
            -
            \frac{1}{n}
            \left(
                t_{i-1} - M_{T_{i-1,n-1}}
            \right)
        \right]
    \\
    ={}&
        S(Q_{i,n-1},T_{i,n-1})
        -
        \frac{1}{n}
        \left(
            t_{i-1} - M_{T_{i-1,n-1}}
        \right)
        \sum \limits _{0 \leq j \lt n-1}
            \left(
                q_{i+j}
                -
                \mu_{Q_{i,n-1}}
            \right)
        -
        \frac{1}{n}
        \left(
            q_{i-1} - \mu_{Q_{i-1,n-1}}
        \right)
        \sum \limits _{0 \leq j \lt n-1}
            \left(
                t_{i+j}
                -
                M_{T_{i,n-1}}
            \right)
        +
        \frac{n-1}{n^2}
        \left(
            q_{i-1} - \mu_{Q_{i-1,n-1}}
        \right)
        \left(
            t_{i-1} - M_{T_{i-1,n-1}}
        \right)
        \\
        &+
        \left[
            \left(
                q_{i-1}
                -
                \mu_{Q_{i,n-1}}
            \right)
            -
            \frac{1}{n}
            \left(
                q_{i-1} - \mu_{Q_{i-1,n-1}}
            \right)
        \right]
        \left[
            \left(
                t_{i-1}
                -
                M_{T_{i,n-1}}
            \right)
            -
            \frac{1}{n}
            \left(
                t_{i-1} - M_{T_{i-1,n-1}}
            \right)
        \right]
    \\
    ={}&
        \sum \limits _{0 \leq j \lt n-1}
            \left(
                q_{i+j}
                -
                \mu_{Q_{i,n-1}}
            \right)
            \left(
                t_{i+j}
                -
                M_{T_{i,n-1}}
            \right)
        -
        \frac{1}{n}
        \left(
            t_{i-1} - M_{T_{i-1,n-1}}
        \right)
        \cdot 0
        -
        \frac{1}{n}
        \left(
            q_{i-1} - \mu_{Q_{i-1,n-1}}
        \right)
        \cdot 0
        +
        \frac{n-1}{n^2}
        \left(
            q_{i-1} - \mu_{Q_{i-1,n-1}}
        \right)
        \left(
            t_{i-1} - M_{T_{i-1,n-1}}
        \right)
        \\
        &+
        \left[
            \left(
                q_{i-1}
                -
                \mu_{Q_{i,n-1}}
            \right)
            -
            \frac{1}{n}
            \left(
                q_{i-1} - \mu_{Q_{i-1,n-1}}
            \right)
        \right]
        \left[
            \left(
                t_{i-1}
                -
                M_{T_{i,n-1}}
            \right)
            -
            \frac{1}{n}
            \left(
                t_{i-1} - M_{T_{i-1,n-1}}
            \right)
        \right]
    \\
    ={}&
        \sum \limits _{0 \leq j \lt n-1}
            \left(
                q_{i+j}
                -
                \mu_{Q_{i,n-1}}
            \right)
            \left(
                t_{i+j}
                -
                M_{T_{i,n-1}}
            \right)
        +
        \frac{n-1}{n^2}
        \left(
            q_{i-1} - \mu_{Q_{i-1,n-1}}
        \right)
        \left(
            t_{i-1} - M_{T_{i-1,n-1}}
        \right)
        \\
        &+
        \left(
            q_{i-1}
            -
            \mu_{Q_{i,n-1}}
        \right)
        \left(
            t_{i-1}
            -
            M_{T_{i,n-1}}
        \right)
        -
        \left(
            q_{i-1}
            -
            \mu_{Q_{i,n-1}}
        \right)
        \frac{1}{n}
        \left(
            t_{i-1} - M_{T_{i-1,n-1}}
        \right)
        -
        \frac{1}{n}
        \left(
            q_{i-1} - \mu_{Q_{i-1,n-1}}
        \right)
        \left(
            t_{i-1}
            -
            M_{T_{i,n-1}}
        \right)
        +
        \frac{1}{n^2}
        \left(
            q_{i-1} - \mu_{Q_{i-1,n-1}}
        \right)
        \left(
            t_{i-1} - M_{T_{i-1,n-1}}
        \right)
    \\
    ={}&
        S(Q_{i,n-1},T_{i,n-1})
        +
        \frac{n-1}{n^2}
        \left(
            q_{i-1} - \mu_{Q_{i-1,n-1}}
        \right)
        \left(
            t_{i-1} - M_{T_{i-1,n-1}}
        \right)
        \\
        &+
        \left(
            q_{i-1}
            -
            \mu_{Q_{i,n-1}}
        \right)
        \left(
            t_{i-1}
            -
            M_{T_{i,n-1}}
        \right)
        -
        \frac{1}{n}
        \left(
            q_{i-1}
            -
            \mu_{Q_{i,n-1}}
        \right)
        \left(
            t_{i-1} - M_{T_{i-1,n-1}}
        \right)
        -
        \frac{n}{n^2}
        \left(
            q_{i-1} - \mu_{Q_{i-1,n-1}}
        \right)
        \left(
            t_{i-1}
            -
            M_{T_{i,n-1}}
        \right)
        +
        \frac{1}{n^2}
        \left(
            q_{i-1} - \mu_{Q_{i-1,n-1}}
        \right)
        \left(
            t_{i-1} - M_{T_{i-1,n-1}}
        \right)
    \\
    ={}&
        S(Q_{i,n-1},T_{i,n-1})
        +
        \frac{n-1}{n^2}
        \left(
            q_{i-1} - \mu_{Q_{i-1,n-1}}
        \right)
        \left(
            t_{i-1} - M_{T_{i-1,n-1}}
        \right)
        -
        \frac{n}{n^2}
        \left(
            q_{i-1} - \mu_{Q_{i-1,n-1}}
        \right)
        \left(
            t_{i-1}
            -
            M_{T_{i,n-1}}
        \right)
        +
        \frac{1}{n^2}
        \left(
            q_{i-1} - \mu_{Q_{i-1,n-1}}
        \right)
        \left(
            t_{i-1} - M_{T_{i-1,n-1}}
        \right)
        \\
        &+
        \frac{n}{n}
        \left(
            q_{i-1}
            -
            \mu_{Q_{i,n-1}}
        \right)
        \left(
            t_{i-1}
            -
            M_{T_{i,n-1}}
        \right)
        -
        \frac{1}{n}
        \left(
            q_{i-1}
            -
            \mu_{Q_{i,n-1}}
        \right)
        \left(
            t_{i-1} - M_{T_{i-1,n-1}}
        \right)
    \\
    ={}&
        S(Q_{i,n-1},T_{i,n-1})
        +
        \frac{n-1-n+1}{n^2}
        \left(
            q_{i-1} - \mu_{Q_{i-1,n-1}}
        \right)
        \left(
            t_{i-1} - M_{T_{i-1,n-1}}
        \right)
        +
        \frac{n-1}{n}
        \left(
            q_{i-1}
            -
            \mu_{Q_{i,n-1}}
        \right)
        \left(
            t_{i-1}
            -
            M_{T_{i,n-1}}
        \right)
    \\
    ={}&
        S(Q_{i,n-1},T_{i,n-1})
        +
        0 \cdot
        \left(
            q_{i-1} - \mu_{Q_{i-1,n-1}}
        \right)
        \left(
            t_{i-1} - M_{T_{i-1,n-1}}
        \right)
        +
        \frac{n-1}{n}
        \left(
            q_{i-1}
            -
            \mu_{Q_{i,n-1}}
        \right)
        \left(
            t_{i-1}
            -
            M_{T_{i,n-1}}
        \right)
    \\
    ={}&
        S(Q_{i,n-1},T_{i,n-1})
        +
        \frac{n-1}{n}
        \left(
            q_{i-1}
            -
            \mu_{Q_{i,n-1}}
        \right)
        \left(
            t_{i-1}
            -
            M_{T_{i,n-1}}
        \right)
    \\
    ={}&
        S(Q_{i,n-1},T_{i,n-1})
        +
        \frac{n-1}{n}
        \left(
            q_{i-1}
            -
            \mu_{Q_{i,n-1}}
        \right)
        \left(
            t_{i-1}
            -
            M_{T_{i,n-1}}
        \right)
    \\
S(Q_{i-1,m}, T_{i-1,m})
    ={}&
        S(Q_{i,m-1},T_{i,m-1})
        +
        \frac{m-1}{m}
        \left(
            q_{i-1}
            -
            \mu_{Q_{i,m-1}}
        \right)
        \left(
            t_{i-1}
            -
            M_{T_{i,m-1}}
        \right)
    \\
\end{align}

### Deriving Rolling Window Variance (or Standard Deviation) with Welford's Method

For any sequence $T$, we can compute the mean, $\mu_{T_{i,n}}$, for any subsequence starting at position $i$ in $T$ and with subsequence length $n$ according to the following definition:

\begin{align}
    \mu_{T_{i,n}} ={}&
        \sum \limits _{0 \leq j \lt n} \frac{t_{i+j}}{n}
    \\
    ={}&
        \frac{t_{i}}{n} + \frac{t_{i+1}}{n} + \ldots + \frac{t_{i+n-2}}{n} + \frac{t_{i+n-1}}{n}
    \\
    ={}&
        0 + \frac{t_{i}}{n} + \frac{t_{i+1}}{n} + \ldots + \frac{t_{i+n-2}}{n} + \frac{t_{i+n-1}}{n}
    \\
    ={}&
        \left(\frac{t_{i-1}}{n} - \frac{t_{i-1}}{n}\right) + \frac{t_{i}}{n} + \frac{t_{i+1}}{n} + \ldots + \frac{t_{i+n-2}}{n} + \frac{t_{i+n-1}}{n}
    \\
    ={}&
        \frac{t_{i-1}}{n} + \frac{t_{i}}{n} + \frac{t_{i+1}}{n} + \ldots + \frac{t_{i+n-2}}{n} + \frac{t_{i+n-1}}{n} - \frac{t_{i-1}}{n}
    \\
    ={}&
        \sum \limits _{-1 \leq j \lt n-1} \frac{t_{i+j}}{n}
        + \frac{t_{i+n-1}}{n} - \frac{t_{i-1}}{n}
    \\
    ={}&
        \mu_{T_{i-1,n}} + \frac{t_{i+n-1} - t_{i-1}}{n}
    \\
    ={}&
        \mu_{T_{i-1,n}} + \frac{t_{i+n-1} - t_{i-1}}{n}
    \\
\end{align}

Then, it follows that:

\begin{align}
    t_{i+j} - \mu_{T_{i,n}} ={}&
        t_{i+j} - \mu_{T_{i-1,n}} - \frac{t_{i+n-1} - t_{i-1}}{n}
    \\
\end{align}

Finally, we can derive the rolling window variance, $S$, by first calculating the correctect sum-of-squares, $CSS$:

\begin{align}
    CSS(T_{i,n}) ={}&
        \sum \limits _{0 \leq j \lt n}
            \left(
                t_{i+j} - \mu_{T_{i,n}}
            \right) ^2
    \\
    ={}&
        \sum \limits _{0 \leq j \lt n-1}
            \left(
                t_{i+j} - \mu_{T_{i,n}}
            \right) ^2
        + \left(
            t_{i+n-1} - \mu_{T_{i,n}}
        \right) ^2
    \\
    ={}&
        \sum \limits _{0 \leq j \lt n-1}
            \left(
                t_{i+j} - \mu_{T_{i,n}}
            \right) ^2
        + \left(
            t_{i+n-1} - \mu_{T_{i,n}}
        \right) ^2
        + 0
    \\
    ={}&
        \sum \limits _{0 \leq j \lt n-1}
            \left(
                t_{i+j} - \mu_{T_{i,n}}
            \right) ^2
        + \left(
            t_{i+n-1} - \mu_{T_{i,n}}
        \right) ^2
        + \left(
            t_{i-1} - \mu_{T_{i,n}}
        \right) ^2
        - \left(
            t_{i-1} - \mu_{T_{i,n}}
        \right) ^2
    \\
    ={}&
        \sum \limits _{0 \leq j \lt n-1}
            \left(
                t_{i+j} - \mu_{T_{i,n}}
            \right) ^2
            + \left(
                t_{i-1} - \mu_{T_{i,n}}
            \right) ^2
        + \left(
            t_{i+n-1} - \mu_{T_{i,n}}
        \right) ^2
        - \left(
            t_{i-1} - \mu_{T_{i,n}}
        \right) ^2
    \\
    ={}&
        \sum \limits _{-1 \leq j \lt n-1}
            \left(
                t_{i+j} - \mu_{T_{i,n}}
            \right) ^2
        + \left(
            t_{i+n-1} - \mu_{T_{i,n}}
        \right) ^2
        - \left(
            t_{i-1} - \mu_{T_{i,n}}
        \right) ^2
    \\
    ={}&
        \sum \limits _{-1 \leq j \lt n-1}
            \left(
                t_{i+j} - \mu_{T_{i-1,n}} - \frac{t_{i+n-1} - t_{i-1}}{n}
            \right) ^2
        + \left(
            t_{i+n-1} - \mu_{T_{i,n}}
        \right) ^2
        - \left(
            t_{i-1} - \mu_{T_{i,n}}
        \right) ^2
    \\
    ={}&
        \sum \limits _{-1 \leq j \lt n-1}
            \left[
                \left(
                    t_{i+j} - \mu_{T_{i-1,n}}
                \right)
                - \frac{t_{i+n-1} - t_{i-1}}{n}
            \right] ^2
        + \left(
            t_{i+n-1} - \mu_{T_{i,n}}
        \right) ^2
        - \left(
            t_{i-1} - \mu_{T_{i,n}}
        \right) ^2
    \\
    ={}&
        \sum \limits _{-1 \leq j \lt n-1}
            \left[
                \left(
                    t_{i+j} - \mu_{T_{i-1,n}}
                \right)^2
                - 2 \left(t_{i+n-1} - t_{i-1} \right)\frac{1}{n}
                \left(
                    t_{i+j} - \mu_{T_{i-1,n}}
                \right)
                + \frac{\left(t_{i+n-1} - t_{i-1}\right)^2}{n^2}
            \right]
        + \left(
            t_{i+n-1} - \mu_{T_{i,n}}
        \right) ^2
        - \left(
            t_{i-1} - \mu_{T_{i,n}}
        \right) ^2
    \\
    ={}&
        \sum \limits _{-1 \leq j \lt n-1}
            \left(
                t_{i+j} - \mu_{T_{i-1,n}}
            \right)^2
            -
        \sum \limits _{-1 \leq j \lt n-1}
            2 \left(t_{i+n-1} - t_{i-1} \right)\frac{1}{n}
            \left(
                t_{i+j} - \mu_{T_{i-1,n}}
            \right)
            +
        \sum \limits _{-1 \leq j \lt n-1}
            \frac{\left(t_{i+n-1} - t_{i-1}\right)^2}{n^2}
        + \left(
            t_{i+n-1} - \mu_{T_{i,n}}
        \right) ^2
        - \left(
            t_{i-1} - \mu_{T_{i,n}}
        \right) ^2
    \\
    ={}&
        \sum \limits _{-1 \leq j \lt n-1}
            \left(
                t_{i+j} - \mu_{T_{i-1,n}}
            \right)^2
        - 2 \left(t_{i+n-1} - t_{i-1} \right)\frac{1}{n}
        \sum \limits _{-1 \leq j \lt n-1}
            \left(
                t_{i+j} - \mu_{T_{i-1,n}}
            \right)
        +
        \sum \limits _{-1 \leq j \lt n-1}
            \frac{\left(t_{i+n-1} - t_{i-1}\right)^2}{n^2}
        + \left(
            t_{i+n-1} - \mu_{T_{i,n}}
        \right) ^2
        - \left(
            t_{i-1} - \mu_{T_{i,n}}
        \right) ^2
    \\
    ={}&
        \sum \limits _{-1 \leq j \lt n-1}
            \left(
                t_{i+j} - \mu_{T_{i-1,n}}
            \right)^2
        - 2 \left(t_{i+n-1} - t_{i-1} \right) \cdot 0
        +
        \frac{\left(t_{i+n-1} - t_{i-1}\right)^2}{n}
        + \left(
            t_{i+n-1} - \mu_{T_{i,n}}
        \right) ^2
        - \left(
            t_{i-1} - \mu_{T_{i,n}}
        \right) ^2
    \\
    ={}&
        CSS(T_{i-1,n})
        +
        \frac{\left(t_{i+n-1} - t_{i-1}\right)^2}{n}
        + \left(
            t_{i+n-1} - \mu_{T_{i,n}}
        \right) ^2
        - \left(
            t_{i-1} - \mu_{T_{i,n}}
        \right) ^2
    \\
    ={}&
        CSS(T_{i-1,n})
        +
        \frac{\left(t_{i+n-1} - t_{i-1}\right)^2}{n}
        + \left(
            t_{i+n-1}^2 - 2t_{i+n-1}\mu_{T_{i,n}} + \mu_{T_{i,n}}^2
        \right)
        - \left(
            t_{i-1}^2 - 2t_{i-1}\mu_{T_{i,n}} + \mu_{T_{i,n}}^2
        \right)
    \\
    ={}&
        CSS(T_{i-1,n})
        +
        \frac{\left(t_{i+n-1} - t_{i-1}\right)^2}{n}
        + t_{i+n-1}^2 - 2t_{i+n-1}\mu_{T_{i,n}} + \mu_{T_{i,n}}^2
        - t_{i-1}^2 + 2t_{i-1}\mu_{T_{i,n}} - \mu_{T_{i,n}}^2
    \\
    ={}&
        CSS(T_{i-1,n})
        +
        \frac{\left(t_{i+n-1} - t_{i-1}\right)^2}{n}
        + t_{i+n-1}^2 - 2t_{i+n-1}\mu_{T_{i,n}} - t_{i-1}^2 + 2t_{i-1}\mu_{T_{i,n}}
    \\
    ={}&
        CSS(T_{i-1,n})
        +
        \frac{\left(t_{i+n-1} - t_{i-1}\right)^2}{n}
        +
        \left(
            t_{i+n-1}^2 - t_{i-1}^2
        \right)
        -
        \left(
            2t_{i+n-1}\mu_{T_{i,n}} - 2t_{i-1}\mu_{T_{i,n}}
        \right)
    \\
    ={}&
        CSS(T_{i-1,n})
        +
        \frac{\left(t_{i+n-1} - t_{i-1}\right)^2}{n}
        +
        \left(
            t_{i+n-1} - t_{i-1}
        \right)
        \left(
            t_{i+n-1} + t_{i-1}
        \right)
        - 2\mu_{T_{i,n}} \left( t_{i+n-1} - t_{i-1} \right)
    \\
    ={}&
        CSS(T_{i-1,n})
        +
        \left(t_{i+n-1} - t_{i-1}\right)
        \left[
            \frac{t_{i+n-1} - t_{i-1}}{n}
            +
            \left(
                t_{i+n-1} + t_{i-1}
            \right)
            - 2\mu_{T_{i,n}}
        \right]
    \\
    ={}&
        CSS(T_{i-1,n})
        +
        \left(t_{i+n-1} - t_{i-1}\right)
        \left[
            \frac{t_{i+n-1} - t_{i-1}}{n}
            +
            \left(
                t_{i+n-1} + t_{i-1}
            \right)
            - \mu_{T_{i,n}}
            - \mu_{T_{i,n}}
        \right]
    \\
    ={}&
        CSS(T_{i-1,n})
        +
        \left(t_{i+n-1} - t_{i-1}\right)
        \left[
            \frac{t_{i+n-1} - t_{i-1}}{n}
            +
            \left(
                t_{i+n-1} - \mu_{T_{i,n}}
            \right)
            +
            \left(
                t_{i-1} - \mu_{T_{i,n}}
            \right)
        \right]
    \\
    ={}&
        CSS(T_{i-1,n})
        +
        \left(t_{i+n-1} - t_{i-1}\right)
        \left[
            \frac{t_{i+n-1} - t_{i-1}}{n}
            +
            \left(
                t_{i+n-1} - \mu_{T_{i,n}}
            \right)
            +
            \left(
                t_{i-1} - \mu_{T_{i-1,n}} - \frac{t_{i+n-1} - t_{i-1}}{n}
            \right)
        \right]
    \\
    ={}&
        CSS(T_{i-1,n})
        +
        \left(t_{i+n-1} - t_{i-1}\right)
        \left[
            \frac{t_{i+n-1} - t_{i-1}}{n}
            -
            \frac{t_{i+n-1} - t_{i-1}}{n}
            +
            \left(
                t_{i+n-1} - \mu_{T_{i,n}}
            \right)
            +
            \left(
                t_{i-1} - \mu_{T_{i-1,n}}
            \right)
        \right]
    \\
    ={}&
        CSS(T_{i-1,n})
        +
        \left(t_{i+n-1} - t_{i-1}\right)
        \left[
            t_{i+n-1} - \mu_{T_{i,n}} + t_{i-1} - \mu_{T_{i-1,n}}
        \right]
    \\
    S^2(T_{i,n}) ={}&
        \frac{\sum \limits _{0 \leq j \lt n}
            \left(
                t_{i+j} - \mu_{T_{i,n}}
            \right) ^2
        }{n}
    \\
    ={}&
        \frac{CSS(T_{i,n})}{n}
    \\
    ={}&
        \frac{CSS(T_{i-1,n})}{n}
        +
        \frac{\left(t_{i+n-1} - t_{i-1}\right)
        \left[
            t_{i+n-1} - \mu_{T_{i,n}} + t_{i-1} - \mu_{T_{i-1,n}}
        \right]}{n}
    \\
    ={}&
        S^2(T_{i-1,n})
        +
        \frac{\left(t_{i+n-1} - t_{i-1}\right)
        \left[
            t_{i+n-1} - \mu_{T_{i,n}} + t_{i-1} - \mu_{T_{i-1,n}}
        \right]}{n}
    \\
\end{align}


- - - -
# Mechpy Tutorials
a mechanical engineering toolbox

source code - https://github.com/nagordon/mechpy
documentation - https://nagordon.github.io/mechpy/web/

- - - -

Neal Gordon
2017-02-20

- - - -

## Sundries

this is just various codes for doing silly things with data processing and engineering stuff

## Convert a string of numbers to a list or array

```python
import numpy as np
y = '''62606.53409
59989.34659
62848.01136
80912.28693
79218.03977
81242.1875
59387.27273
73027.5974
69470.69805
66843.99351
82758.44156
81647.72727
77519.96753'''
y = [float(x) for x in np.array(y.replace('\n',',').split(','))]
print(y, end=" ")
```

    [62606.53409, 59989.34659, 62848.01136, 80912.28693, 79218.03977, 81242.1875, 59387.27273, 73027.5974, 69470.69805, 66843.99351, 82758.44156, 81647.72727, 77519.96753]

## Ascii Plot

```python
from sympy import symbols, exp, pi, cos
from sympy.plotting import textplot
x = symbols('x')
textplot( exp(-x) * cos(2*pi*x)  ,0,5, 75,20)
```

          1 |
            |  .
            |
            |
            |
            |   .
            |
            |               ..
            |              .  \
    0.19330 | ---.-------------.---------------------------------------------------------
            |             .                ...
            |                   .        ..   ..         ....
            |                    .      /       ..    ...    ............................
            |     .      .             /          ....
            |                     ..  /
            |           .           ..
            |      .
            |          .
            |
    -0.6133 |       ...
              0                                2.5                                  5

## another example of an Ascii Plot

```python
# This is the same example as in the scitools.aplotter doc string

import numpy as np
x = np.linspace(0, 5, 81)
#y = np.exp(-0.5*x**2)*np.cos(np.pi*x)
y=np.exp(-x)*np.cos(2*np.pi*x)
from scitools.aplotter import plot
plot(x, y)

plot(x, y, draw_axes=False)

# Plot symbols (the dot argument) at data points
plot(x, y, plot_slope=False)

# Drop axis labels
plot(x, y, plot_labels=False)

plot(x, y, dot='o', plot_slope=False)

# Store plot in a string
p = plot(x, y, output=str)
print(p)
```


[](https://colab.research.google.com/github/comp-hci-lab/BDSI_2021_ML/blob/master/8%20-%20Hyperparameter%20%26%20Model%20Selection.ipynb)

## BDSI ML 2021

# 8 - Hyperparameter & Model Selection

Consider the objective function of Linear SVM:

\begin{equation}
J(\bar\theta) = \frac{1}{2}\|\bar\theta\|_2^2 + C\sum_{i=1}^{N} \max\{0, 1 - y^{(i)}\bar{\theta}\cdot \bar{x}^{(i)}\}
\end{equation}

Given training data, there are two sets of unknowns:
- Model parameter(s): the coefficients $\bar\theta$, which are found by fitting the model,
- Hyperparameter(s): the regularization strength (or its inverse, $C$), which has to be specified prior to the fitting the model.

**Note:** Model parameters can be determined through a learning algorithm (e.g., SGD, or calling `fit(X, y)`. But how do we select the best hyperparameters?

```python
#@title Run this cell to download preprocessed data (features + labels). { display-mode: "form" }
!pip install -U wget
!rm -rf preprocessed
!mkdir preprocessed

import wget
wget.download('https://github.com/comp-hci-lab/BDSI_2021_ML/raw/master/preprocessed/data.npz', 'preprocessed/data.npz')
```

```python
import numpy as np
import pandas as pd
from tqdm import tqdm
import matplotlib.pyplot as plt
from sklearn.svm import SVC
from sklearn import metrics, exceptions
```

```python
with np.load('preprocessed/data.npz') as f:
    X = f['X']
    y = f['y']
    feature_names = f['feature_names']
```

```python
#@title Run this cell to define the preprocessing functions. { display-mode: "form" }
#@markdown - `impute_missing_values(X)`
#@markdown - `normalize_feature_matrix(X)`

def impute_missing_values(X):
    """
    For each feature column, impute missing values  (np.nan) with the
    population mean for that feature.

    Args:
        X: np.array, shape (N, d). X could contain missing values
    Returns:
        X: np.array, shape (N, d). X does not contain any missing values
    """
    from sklearn.impute import SimpleImputer
    return SimpleImputer().fit_transform(X)

def normalize_feature_matrix(X):
    """
    For each feature column, normalize all values to range [0, 1].

    Args:
        X: np.array, shape (N, d).
    Returns:
        X: np.array, shape (N, d). Values are normalized per column.
    """
    from sklearn.preprocessing import MinMaxScaler
    return MinMaxScaler().fit_transform(X)
```

```python
X = impute_missing_values(X)
X = normalize_feature_matrix(X)
```

```python
print('First 10 labels:', y[:10])
print('First 2 feature vectors:\n', X[:2])
```

## Pipeline v0

```python
clf = SVC(kernel='linear') # use default arguments here
clf.fit(X, y)
y_pred = clf.predict(X)
accuracy = metrics.accuracy_score(y, y_pred)
print('Accuracy:', accuracy)
```

#@title What's wrong with Pipeline v0? How do we fix it? { display-mode: "form" }
- answer =  #@param {type:"raw"}

## Pipeline v1

```python
# Split data into train (80%) and test (20%)
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, stratify=y, random_state=3)
```

```python
clf = SVC(kernel='linear')
clf.fit(...)
y_pred = clf.predict(...)
accuracy = metrics.accuracy_score(...)
print('test accuracy:', accuracy)
```

```python
#@title What's wrong with Pipeline v1? How do we fix it? { display-mode: "form" }
answer =  #@param {type:"raw"}
```

## Pipeline v2

```python
for C in [1e-1, 1, 1e1]:
    clf = SVC(kernel='linear', C=C)
    clf.fit(X_train, y_train)
    y_pred = clf.predict(X_test)
    accuracy = metrics.accuracy_score(y_test, y_pred)
    print('C={}'.format(C), '\t', 'test accuracy:', accuracy)
# in the end, pick the C with best test accuracy
```

#@title What's wrong with Pipeline v2? How do we fix it? { display-mode: "form" }
- answer =  #@param {type:"raw"}

---
## Pipeline v3: Hyperparameter Selection via Cross validation

We will select hyperparameters using 5-fold cross-validation (CV) on the training data. Specifically, we will select the hyperparameters that lead to the best average validation performance across all five folds. The result of hyperparameter selection often depends on the choice of performance measure.

### (a)
First, implement the function `cv_performance(clf, X, y, metric='accuracy', k=5)` to calculate cross-validated performance given a classifier and (training) data `X, y`.

When dividing the data into folds for CV, you should try to keep the class proportions (ratio of positive to negative labels) roughly the same across folds. You may employ the following class for splitting the data: `sklearn.model_selection.StratifiedKFold()`. For consistency of results, do not shuffle points when using this function (i.e., do not set `shuffle=True`).

#@title Why might it be beneficial to maintain class proportions across folds? { display-mode: "form" }
- answer = ' ' #@param {type:"raw"}

```python
from sklearn.model_selection import KFold, StratifiedKFold

def cv_performance(clf, X, y, metric='accuracy', k=5):
    """
    Splits the training data X and the labels y into k-folds and runs k-fold
    cross-validation: for each fold i in 1...k, trains a classifier on
    all the data except the ith fold, and tests on the ith fold.

    Calculates the k-fold cross-validation performance metric for classifier
    clf by averaging the performance across folds.

    Input:
        clf: a classifier
        X: (n,d) array of feature vectors, where n is the number of examples
           and d is the number of features
        y: (n,) array of binary labels
        k: an int specifying the number of folds (default=5)
        metric: string specifying the performance metric

    Returns:
        average validation performance across the k folds
    """
    # TODO: Implement this function

    ## HINT: You may find the StratifiedKFold from sklearn.model_selection
    ## to be useful

    # Put the performance of the model on each fold in the scores array
    scores = []

    # And return the average performance across all fold splits.
    return np.array(scores).mean()
```

### (b)
Now implement the `select_C_linear_SVM(X, y, metric='accuracy', k=5, C_range=[])` function to choose a value of C for a linear SVM, using 5-fold cross validation on the training data with specified metric. This function should call the `cv_performance` function that you implemented above, passing in instances of `SVC(kernel='linear', C=C)` with a range of values for C chosen in powers of 10 (between $10^{−3}$ and $10^3$).

```python
def select_C_linear_SVM(X, y, metric='accuracy', k=5, C_range=[]):
    """
    Sweeps different settings for the hyperparameter of a linear SVM,
    calculating the k-fold CV performance for each setting of C on
    training data X, y.

    Input:
        X: (n,d) array of feature vectors, where n is the number of examples
        and d is the number of features
        y: (n,) array of binary labels
        k: int specifying the number of folds (default=5)
        metric: string specifying the performance metric
        C_range: an array with C values to be searched over

    Returns:
        The value of C parameter for a linear SVM that maximizes the
        average 5-fold CV performance.
    """
    # TODO: Implement this function
    #HINT: You should be using your cv_performance function here
    #to evaluate the performance of each SVM
    return 0.0
```

### (c)

For each metric, describe how the validation performance varies with C (e.g., make a plot).

If you need to train a final model, which performance measure would you optimize for when choosing $C$? Explain your choice.

Using the training data and functions implemented here, find the best setting for $C$ for your chosen performance measure. Using the best $C$ value you found, train a SVM on the entire training set `X_train, y_train`. Report the performance of this SVM on the test data `X_test, y_test` for each metric below.

```python
## TODO
metric_list = ["accuracy", "f1_score", "auroc", "precision", "sensitivity", "specificity"]

```

```python
## TODO
# Select the best C based on your chosen metric

# Train a model using best C on entire training set

# Evaluate classifier performance on test set

```

| Metric |  Best C | Test Performance |
|------|------|------|
| accuracy | ? | ? |
| F1-score | ? | ? |
| AUROC    | ? | ? |
| Precision | ? | ? |
| Sensitivity | ? | ? |
| Specificity | ? | ? |

### (d)
The L0-norm of $\bar{\theta} \in \mathbb{R}^d$ is defined as:
\begin{align*}
    \|\bar{\theta}\|_0 = \sum_{j=1}^{d} [[\theta_j \neq 0]]
\end{align*}
where $[[a \neq 0]]$ is 0 if $a$ is 0 and 1 otherwise.

Plot the L0-norm of $\bar{\theta}$, the parameter vector learned by the SVM, for each value of $C$. Use the complete training data `X_train, y_train`, i.e, don't use cross-validation for this part.

```python
from sklearn.svm import SVC, LinearSVC
def get_classifier(kernel='linear', penalty='l2', C=1.0, gamma=0.0, class_weight=None):
    """
    Return a linear/rbf kernel SVM classifier based on the given
    penalty function and regularization parameter c.
    """
    if penalty == 'l2':
        if kernel == 'linear':
            return SVC(kernel='linear', C=C, class_weight=class_weight)
        elif kernel == 'rbf':
            return SVC(kernel='rbf', C=C, gamma=gamma, class_weight=class_weight)
    elif penalty == 'l1':
        return LinearSVC(penalty='l1', C=C, dual=False, max_iter=20000, class_weight=class_weight)

    raise ValueError('Error: unsupported configuration')

def plot_weight(X, y, penalty, C_range):
    """
    Takes as input the training data X and labels y and plots the L0-norm
    (number of nonzero elements) of the coefficients learned by a classifier
    as a function of the C-values of the classifier.
    """
    print("Plotting the number of nonzero entries of the parameter vector as a function of C")
    norm0 = []

    ### Solution
    for C in C_range:
        clf = get_classifier(kernel='linear', C=C, penalty=penalty)
        clf.fit(X, y)
        w = clf.coef_
        w = np.squeeze(np.asarray(w))
        norm0.append(np.linalg.norm((w),ord=0))
    ### Solution

    # This code will plot your L0-norm as a function of C
    fig, ax = plt.subplots()
    plt.plot(C_range, norm0)
    plt.xscale('log')
    plt.legend(['L0-norm'])
    plt.xlabel("Value of C")
    plt.ylabel("Norm of theta")
    plt.title('Norm-'+penalty+'_penalty.png')
    return fig

```

```python
# L2 regularization
fig = plot_weight(X_train, y_train, 'l2', np.logspace(-5, 2, 8))
plt.ylim(0-1, 40+1)
plt.grid()
plt.show()
```

```python
# L1 regularization
fig = plot_weight(X_train, y_train, 'l1', np.logspace(-5, 2, 8))
plt.ylim(0-1, 40+1)
plt.grid()
plt.show()
```

#@title Describe any interesting trends you observe. { display-mode: "form" }
- answer =  #@param {type:"raw"}

```python

```

8 - Hyperparameter & Model Selection.ipynb