```python
from natsort import natsorted
import matplotlib.pyplot as plt
from glob import glob
import json
from functools import partial
import numpy as np
from cachier import cachier
from tqdm import tqdm
import imageio
from skimage import img_as_float32
import cv2 as cv2
from math import pi, e, sqrt, cos, sin
from scipy.stats import norm, multivariate_normal
from scipy.spatial.distance import euclidean
from sklearn.metrics import accuracy_score
from skimage.feature import hog
from sklearn.decomposition import PCA
import pandas as pd
```

```python
ztqdm = partial(tqdm, position=0, leave=True)
cachier = partial(cachier, pickle_reload=False, cache_dir='data/cache')
```

```python
SIZE = (768, 1024)
NEW_SIZE = SIZE[::-1]
DATA_PATH_PAIRS = list(zip(
    natsorted(glob(f'../data/images-{SIZE[1]}x{SIZE[0]}/*.png')),
    natsorted(glob(f'../data/masks-{SIZE[1]}x{SIZE[0]}/*.png')),
))
DATA_IMGS = np.array(
    [cv2.resize(img_as_float32(imageio.imread(img_path)), NEW_SIZE)
     for img_path, _ in tqdm(DATA_PATH_PAIRS, 'Loading Images')])
DATA_MSKS = np.array(
    [cv2.resize(img_as_float32(imageio.imread(msk_path)), NEW_SIZE,  interpolation = cv2.INTER_NEAREST)
     for _, msk_path in tqdm(DATA_PATH_PAIRS, 'Loading Masks')])

```

    Loading Images: 100%|██████████████████████████████████████████████████████████████████| 48/48 [00:01<00:00, 28.02it/s]
    Loading Masks: 100%|██████████████████████████████████████████████████████████████████| 48/48 [00:00<00:00, 148.59it/s]

```python
def normalise_data(data):
    return np.subtract(data, np.min(data)) / np.subtract(np.max(data), np.min(data))

def show_image(image,cs=False,cmap= None, title=None):
    if cs:
        norm_img = normalise_data(image)
    else:
        norm_img = image
    plt.title(title)
    plt.imshow(norm_img, cmap= cmap)
    plt.xticks([])
    plt.yticks([])
```

```python
class ImageDataUtils(object):

    def train_test_split(self, images, masks, train_size=0.7, validation_size=0.15, test_size=0.15, seed=5):
        """
        Split a set of images and masks according to a random specified set sizes.

        """
        # Get indices
        np.random.seed(seed)
        all_indices = np.arange(len(images))
        np.random.shuffle(all_indices)
        train_indices = all_indices[0: int(len(images)* train_size) + 1]
        validate_inidices = all_indices[len(train_indices): len(train_indices) + int(len(images)* validation_size)]
        test_indices = all_indices[len(train_indices)+len(validate_inidices): len(train_indices)+
                                   len(validate_inidices) + int(len(images)* test_size)]
        # Train data
        train_images = np.array([images[i] for i in train_indices])
        train_masks = np.array([masks[i] for i in train_indices])
        # Validate data
        validate_images = np.array([images[i] for i in validate_inidices])
        validate_masks = np.array([masks[i] for i in validate_inidices])
        # Test data
        test_images = np.array([images[i] for i in test_indices])
        test_masks = np.array([masks[i] for i in test_indices])
        return train_images, train_masks, validate_images, validate_masks, test_images, test_masks

    def _get_kernel(self, g, sigma, size, k=None):
        """
        Function to map kernel function to numpy matrix,
        Note result is normalized.

        """
        # Center x and y coordinates and return a numpy matrix from g function
        kernel =  np.fromfunction(lambda x, y:
                                 g((x - (size - 1) / 2), (y - (size - 1) / 2), sigma, k),
                                 (size, size))
        return kernel.astype(np.float32)

    def _dog(self, sigma, size, k):
        """
        Get a Difference of Gaussian filter

        """
        def _g(x, y, sigma, k):
            return (2*pi*sigma**2) * \
                    e**(-1 * ((x**2 + y**2) / (2 * sigma**2))) - \
                    (1 / 2 * pi * k**2 * sigma**2) *\
                    e**(-1 * ((x**2 + y**2)/(2 * k**2 * sigma**2)))

        if k <= 1.0:
            raise ValueError("k must be > 1.0")

        return self._get_kernel(_g, sigma, size, k)

    """
    Get a Laplacian of Gaussian kernel

    """
    def _log(self, sigma, size):

        def _g(x, y, sigma, k= None):
            return (1/(pi * sigma**4)) * \
                    (1 - ((x**2 + y**2)/(2 * sigma**2))) * \
                    e ** (-1 * ((x**2 + y**2)/(2 * sigma**2)))

        return self._get_kernel(_g, sigma, size)

    def extract_features(self, image, gaus=(3,3), dog_sigma=30, dog_size=65, dog_k=6):
        """
        Parameters
        ----------
        image : numpy array of shape (n_sample, col, row, n_channels)
        gaus : tuple of gaussian kernel for blurring input image
        dog_sigma : int, sigma value for DoG features
        dog_size  : int, size of DoG kernel
        dog_k : int, k parameter for DoG
        show_feature_plots : bool, verbose option to plot features

        Returns
        ----------
        numpy array : 9 x flattened image shape (786432) features of image.
        1 - 3: R, G, B channels
        4 - 6: DoG of RGB channels
        7 - 9: H, S, V channels

        """
         # Gaussian blur image as a first step to reduce noise
        image_ = cv2.GaussianBlur(image, gaus, 0).astype(np.float32)
        # RBG Features
        features = None
        for rgb_channel in cv2.split(image_):
            if features is None:
                features = np.array(normalise_data(rgb_channel.flatten().astype(np.float32)))
            else:
                features = np.vstack((features, normalise_data(rgb_channel.flatten().astype(np.float32))))

        dog_kernel = self._dog(dog_sigma, dog_size, dog_k)
        convolved = cv2.filter2D(image_, -1, dog_kernel)
        for convolved_rgb_channel in cv2.split(convolved):
            if features is None:
                features = np.array(normalise_data(convolved_rgb_channel.flatten().astype(np.float32)))
            else:
                features = np.vstack((features, normalise_data(convolved_rgb_channel.flatten().astype(np.float32))))

        return features

    def apply_mask(self, features, mask, invert_mask=False):
        """Apply the mask to a data set"""
        data = []
        mask_ = mask.astype(np.bool).flatten()
        if invert_mask:
            mask_ = np.invert(mask_)
        for feature in features:
            masked_feature = feature[mask_ == True]
            data.append(masked_feature.flatten())
        return np.array(data)
```

# Train, validation and training split
Split data randomly into sets of 70% training, 15% validation and 15% testing

```python
data_utils = ImageDataUtils()

train_images, train_masks, \
validate_images, validate_masks, \
test_images, test_masks = data_utils.train_test_split(DATA_IMGS, DATA_MSKS,
                                                      train_size=0.7,
                                                      validation_size=0.15,
                                                      test_size=0.15)
```

# Pre-process data, and extract the features:
Flattened and normalised feature vectors for each image of
- RGB channels
- DoG on RGB channels

The training data is split into two sets (foreground and background) to train two GMM models. Lastly all of the training
data are concatenated into a signle dimension so that we have data sets in the format (n_features, samples)

```python
train_data = np.array([data_utils.extract_features(train_image) for train_image in train_images])
validate_data = np.array([data_utils.extract_features(validate_image) for validate_image in validate_images])
test_data = np.array([data_utils.extract_features(test_image) for test_image in test_images])
```

Use masks to split training data to foreground and background data sets

```python
train_bg_data = [data_utils.apply_mask(features, mask, invert_mask=True)
                for features, mask in zip(train_data, train_masks)]
train_fg_data = [data_utils.apply_mask(features, mask, invert_mask=False)
                for features, mask in zip(train_data, train_masks)]
```

```python
train_bg_data = np.concatenate(train_bg_data, axis=1)
train_fg_data = np.concatenate(train_fg_data, axis=1)
```

## Gaussian Mixture Model (GMM)

Since our data is composed of intensities that might be modelled using gaussian distributions, we investigate the use of a Mixture Gausssian Model (GMM), which is an unsupervised learning approach that aims to fit a mixture of gasussian distributions to model the data with $d$ dimensions of features. And we want to use the GMM to fit clusters or mixture of gaussians to our data set, but we do not know where they are, or how they are shaped.
It differs from normal KNN, since the clusters are gaussian distributions, which are parameterised by a mean and covariance. Which allows us to model the probability that our data belongs to either clusters in elliptical shapes, as opposed to the circular shapes clustered by KNN.

### Expectation Maximisation (EM)
We find the gaussian mixture model by using the EM algorithm to find k to minimize $\frac{(x-u_k)^2}{\sigma^2}$.
EM introduces a latent variable $h$ for each cluster $K$, and the GMM is a marginalisation of the joint probability distribution $P(\vec{x}, h)$, where $\vec{x}$ represents our feature vectors.  We define a variable $h \in \{1 \dots K\}$ and then the probability distribution for each categorical variable is written as

$Pr(\times|h, \theta) = Norm_x[\mu_h, \Sigma_h]$

$Pr(h|\theta) = Cat_h[\lambda]$

Then we can recover the density by marginalising out the $h$ from the joint probability distribution $Pr(\vec{x},h)$

$
\begin{align}
Pr(\vec{x}|\theta) & = \sum^K_{k=1} Pr(\vec{x}, h = k | \theta)\\
                   & = \sum^K_{k=1} Pr(\vec{x}, h = k, \theta) Pr(h=k|\theta)\\
                   & = \sum^K_{k=1} \lambda_k Norm_\vec{x}[\mu_k, \Sigma_h]   \\
\end{align}
$

This formulation allows us to fit the model with the Expectation Maximisation algorithm using closed-form solutions in an iterative process. The goal is to learn the model parameters $\theta = \{\lambda_{1 \dots K}, \mu_{1 \dots K}, \Sigma_{1 \dots K} \}$ from our training features $\vec{x}$. The two steps of the EM algorithm include

#### 1. E-Step (Expectation)
The probability of $h$ is computed using the current estimates of the parameters $\theta$. So for each point we estimate the probability that each Gaussian generated it $Pr(h_i = k | \vec{x}_i, \theta^{[t]})$, which might also be described as the responsibility of the $k^{th}$ Gaussian for the $i^{th}$ data point expressed by

$r_{ik} = \frac{\lambda_k Norm_{x_i} [\mu_k, \Sigma_k]}{\sum^K_{j=1}\lambda_j Norm_\vec{x_i} [\mu_j, \Sigma_j]}$

#### 1. M-Step (Maximisation)
Using the computed resposibilities the maximisation step then updates the model parameters $\theta$. And in our case we use maximum likelihood estimation that allows the following closed form update rules

$\lambda_k^{[t + 1]} = \frac{\sum^I_{i=1} r_{ik}} {\sum^K_{j=1}\sum^I_{i=1} r_{ij}  }   $

$\mu_k^{[t + 1]} = \frac{\sum^I_{i=1} r_{ik} \vec{x_i}} {\sum^I_{i=1} r_{ik}} $

$\Sigma_k^{[t + 1]} = \frac{\sum^I_{i=1} r_{ik} (\vec{x_i} - \mu_k^{[t + 1]}) (\vec{x_i} - \mu_k^{[t + 1]})^T}{\sum^I_{i=1} r_{ik}}$

#### The EM-flow

- The $k$ clusters center are randomly initialised
- The responsibilities are recomputed during the E-Step, and the model parameters are updated during the M-Step.
- The iterative process is repeated until convergence (Parameter change is smaller then some threshold.)

### GMM using EM

```python
class GuassianParameters(object):
    """Distribution parameters"""

    def __init__(self, lmbda, mu, cov):
        self.lmbda = lmbda
        self.mu = mu
        self.cov = cov

    def difference_between(self, other):
        return abs(self.lmbda - other.lmbda) + \
               euclidean(self.mu, other.mu) + \
               euclidean(self.cov.flatten(), other.cov.flatten())

class GaussianMixtureModel(object):
    """Multivariate Gaussian Mixture Model"""

    def _get_init_params(self, X):
        n_features = X.shape[1]
        # Initial weigths evenly distributed
        lmbda = 1 / self.k_size
        # Initial means, random observations from input data
        mu = np.random.choice(X.flatten(), n_features)
        # Iniital covariance
        cov = np.random.random((n_features, n_features))
        cov *= cov.T
        cov += n_features * np.eye(n_features)
        return GuassianParameters(lmbda, mu, cov)

    def __init__(self, k_size=5, tol=1e-3, min_iter=5, max_iter=60, verbose=True):
        self.fitted = False
        self.k_size = k_size
        self.tol = tol
        self.min_iter = min_iter
        self.max_iter = max_iter
        self.verbose = verbose
        self.fitted_parameters = None

    def _estep(self, X, gaussian_parameters):
        """Calculate the responsibility for each cluster for all of the data points,"""
        n_samples = X.shape[0]
        responsibilities = np.zeros((n_samples, self.k_size))
        weighted_gaussians = np.zeros((n_samples, self.k_size))
        # Get normal distributions using current parameter set
        for k in range(self.k_size):
            weighted_gaussians[:, k] = gaussian_parameters[k].lmbda * \
                                       multivariate_normal.pdf(X, gaussian_parameters[k].mu, gaussian_parameters[k].cov)
        # Compute responsibility for each cluster
        for k in range(self.k_size):
            responsibilities[:, k] = weighted_gaussians[:, k] / np.sum(weighted_gaussians, axis=1, initial=1e-10)
        return responsibilities

    def _mstep(self, X, responsibilities):
        """Compute model parameters using maximum likelikhood closed form solutions"""
        gaussian_parameters = []
        for k in range(self.k_size):
            k_sum_responsibility = np.sum(responsibilities[:, k])
            lmbda = k_sum_responsibility / (np.sum(np.sum(responsibilities, axis=1)))
            mu = np.dot(responsibilities[:, k], X) / k_sum_responsibility
            standardised_x = X - mu
            cov = np.dot((standardised_x.T * responsibilities[:, k]), standardised_x) / k_sum_responsibility
            # Add small value to diagonal to prevent singular matrix
            cov += np.eye(len(cov)) * 1e-10
            gaussian_parameters.append(GuassianParameters(lmbda, mu, cov))
        return gaussian_parameters

    def save_parameters(self, filename):
        """Save the model parameters to file"""
        assert self.fitted, "Model is not yet fitted."
        np.save(filename, self.fitted_parameters)

    def load_parameters(self, filename):
        """Load the model parameters from file"""
        self.fitted_parameters = np.load(filename +'.npy',allow_pickle=True)
        self.fitted = True

    def plot_gaussian(self, X, feature_1_index, feature_2_index, component_number):
        """Plots a gaussian distribution for two random variables"""
        assert self.fitted, "Model is not yet fitted."
        n_samples = X.shape[0]
        gaussian_pdfs = np.zeros((n_samples, self.k_size))
        for k in range(self.k_size):
            gaussian_pdfs[:, k] = self.fitted_parameters[k].lmbda *\
            multivariate_normal.pdf(X, self.fitted_parameters[k].mu, self.fitted_parameters[k].cov)
        fig = plt.figure()
        ax = fig.gca(projection='3d')
        ax.plot_surface(X[:, feature_1_index],
                        X[:, feature_2_index],
                        gaussian_pdfs[:, component_number] ,cmap='viridis',linewidth=0)
        ax.set_xlabel('Feature 1')
        ax.set_ylabel('Feature 2')
        ax.set_zlabel('Likelihood')
        plt.show()

    def fit(self, X):
        """Fit the model according to the given training data.

        Parameters
        ----------
        X : numpy array, of shape (n_samples, n_features)
            Training vector, where n_samples is the number of samples/pixels and
            n_features is the number of features.

        """
        converged = False
        run_iteration = 1
        # Initialise parameters
        gaussian_parameters = [self._get_init_params(X) for _k in range(self.k_size)]
        while not converged and run_iteration <= self.max_iter:
            # E-step
            responsibilities = self._estep(X, gaussian_parameters)
            # M-step
            new_gaussian_parameters = self._mstep(X, responsibilities)
            # Iterate until parameters experience change smaller than tolerance
            total_change = 0
            for k, parameters in enumerate(new_gaussian_parameters):
                total_change += parameters.difference_between(gaussian_parameters[k])
            if total_change < self.tol and run_iteration > self.min_iter:
                converged = True
            gaussian_parameters = new_gaussian_parameters
            if self.verbose:
                print("Iteration #{0}, change in parameters {1}".format(run_iteration, total_change))
            run_iteration += 1

        self.fitted_parameters = gaussian_parameters
        self.fitted = True

        if self.verbose:
            print("Finished in {0} iterations".format(run_iteration - 1))

    def predict(self, X):
        """ Predict's the probability density the input observations by marginalising
        out the joint probability distributions.

        Parameters
        ----------
        X : numpy array, of shape (n_samples, n_features)
            Testing vector, where n_samples is the number of samples/pixels and
            n_features is the number of features.

        Returns
        ----------
        y : numpy array, of shape (n_samples)
            Estimated probability density from the mixture of gaussian models.
        """
        assert self.fitted, "Model is not yet fitted."
        n_samples = X.shape[0]
        gaussian_pdfs = np.zeros((n_samples, self.k_size))
        for k in range(self.k_size):
            gaussian_pdfs[:, k] = self.fitted_parameters[k].lmbda *\
            multivariate_normal.pdf(X, self.fitted_parameters[k].mu, self.fitted_parameters[k].cov)
        # Return marginalised distribution
        return np.sum(gaussian_pdfs, axis=1)
```

**Background model**

```python
bg_gmm = GaussianMixtureModel()
bg_gmm.fit(train_bg_data.T)
```

    Iteration #1, change in parameters 79.43740975072042
    Iteration #2, change in parameters 0.012996177606300752
    Iteration #3, change in parameters 0.05620474937803228
    Iteration #4, change in parameters 0.15217696315190313
    Iteration #5, change in parameters 0.16459611788670195
    Iteration #6, change in parameters 0.1801253563594018
    Iteration #7, change in parameters 0.17338621119328285
    Iteration #8, change in parameters 0.16127863504672205
    Iteration #9, change in parameters 0.18812908819012855
    Iteration #10, change in parameters 0.1918986604731479
    Iteration #11, change in parameters 0.16152831159366782
    Iteration #12, change in parameters 0.14854045908629326
    Iteration #13, change in parameters 0.12878702603274825
    Iteration #14, change in parameters 0.10796152600020921
    Iteration #15, change in parameters 0.08238704157607385
    Iteration #16, change in parameters 0.0595499222562206
    Iteration #17, change in parameters 0.050130647123780125
    Iteration #18, change in parameters 0.04436345150232728
    Iteration #19, change in parameters 0.04045598350277784
    Iteration #20, change in parameters 0.03921934349134461
    Iteration #21, change in parameters 0.04225088121104163
    Iteration #22, change in parameters 0.04513150069755301
    Iteration #23, change in parameters 0.030877525122151264
    Iteration #24, change in parameters 0.02451158597988397
    Iteration #25, change in parameters 0.023613428452386877
    Iteration #26, change in parameters 0.019503341260520342
    Iteration #27, change in parameters 0.015038064686111092
    Iteration #28, change in parameters 0.01128552470883976
    Iteration #29, change in parameters 0.008328619317375817
    Iteration #30, change in parameters 0.0060788117151650365
    Iteration #31, change in parameters 0.004425028136590352
    Iteration #32, change in parameters 0.0032312690793125437
    Iteration #33, change in parameters 0.0023924982052138492
    Iteration #34, change in parameters 0.001785954371456925
    Iteration #35, change in parameters 0.0013440235922544692
    Iteration #36, change in parameters 0.0010198464438408118
    Iteration #37, change in parameters 0.0007801482360644345
    Finished in 37 iterations

**Foreground model**

```python
fg_gmm = GaussianMixtureModel()
fg_gmm.fit(train_fg_data.T)
```

    Iteration #1, change in parameters 80.03211249178665
    Iteration #2, change in parameters 0.03378099767418124
    Iteration #3, change in parameters 0.07936074909766108
    Iteration #4, change in parameters 0.21321926222194243
    Iteration #5, change in parameters 0.3985531762798284
    Iteration #6, change in parameters 0.47948086524062483
    Iteration #7, change in parameters 0.3783289749983527
    Iteration #8, change in parameters 0.2794349137987201
    Iteration #9, change in parameters 0.2810133008285232
    Iteration #10, change in parameters 0.3432099936386448
    Iteration #11, change in parameters 0.3853426560621266
    Iteration #12, change in parameters 0.3160843215610648
    Iteration #13, change in parameters 0.17995805608688792
    Iteration #14, change in parameters 0.14105029356580798
    Iteration #15, change in parameters 0.12481403692095244
    Iteration #16, change in parameters 0.11260745375752639
    Iteration #17, change in parameters 0.10308677849261977
    Iteration #18, change in parameters 0.09628679182602906
    Iteration #19, change in parameters 0.09017210324079891
    Iteration #20, change in parameters 0.08425394804444208
    Iteration #21, change in parameters 0.07817840151639496
    Iteration #22, change in parameters 0.07184228583893187
    Iteration #23, change in parameters 0.06522383391631603
    Iteration #24, change in parameters 0.05808204202634088
    Iteration #25, change in parameters 0.05054030349348687
    Iteration #26, change in parameters 0.043119395075037995
    Iteration #27, change in parameters 0.036228337286009916
    Iteration #28, change in parameters 0.03025215785687432
    Iteration #29, change in parameters 0.02528106751986077
    Iteration #30, change in parameters 0.021173001066641682
    Iteration #31, change in parameters 0.017792893996548587
    Iteration #32, change in parameters 0.014926258180491401
    Iteration #33, change in parameters 0.012485741931145974
    Iteration #34, change in parameters 0.01043113803889918
    Iteration #35, change in parameters 0.008719046261628687
    Iteration #36, change in parameters 0.007300638215990642
    Iteration #37, change in parameters 0.006126282703606314
    Iteration #38, change in parameters 0.0051511108134122095
    Iteration #39, change in parameters 0.004338120714050568
    Iteration #40, change in parameters 0.0036579610492355587
    Iteration #41, change in parameters 0.003087375616531636
    Iteration #42, change in parameters 0.002607735129130816
    Iteration #43, change in parameters 0.002203944150768061
    Iteration #44, change in parameters 0.0018636517093872339
    Iteration #45, change in parameters 0.0015766719286707663
    Iteration #46, change in parameters 0.0013345495328648612
    Iteration #47, change in parameters 0.0011302257451215817
    Iteration #48, change in parameters 0.0009577744873276903
    Finished in 48 iterations

**Posterior probabilities**

$P(class=fg\  | \ \vec{x}) =  \frac{\lambda Norm_\vec{x} [\vec{\mu}_{fg} , \Sigma_{fg}]}
                                   {\lambda Norm_\vec{x} [\vec{\mu}_{fg} , \Sigma_{fg}] \ + \ (1 - \lambda)
                                   Norm_\vec{x} [\vec{\mu}_{bg} , \Sigma_{bg}]   }        $

```python
def post_processing(fg_bg_pred, kernel_dims=5):
    """Apply morhpolical transformations of closing
    and opening"""
    kernel = np.ones((kernel_dims, kernel_dims),np.uint8)
    fg_bg_pred = cv2.morphologyEx(fg_bg_pred, cv2.MORPH_OPEN, kernel)
    fg_bg_pred = cv2.morphologyEx(fg_bg_pred, cv2.MORPH_CLOSE, kernel)
    return fg_bg_pred

def predict_foreground(X, bg_gmm, fg_gmm, lmbda):
    """Use Bayes' rule to predict the posterior probability of
    of pixels belonging to the foreground."""
    fg_dist = fg_gmm.predict(X)
    bg_dist = bg_gmm.predict(X)
    y = (lmbda * fg_dist) / ((lmbda * fg_dist) + ((1 - lmbda) * bg_dist))
    return post_processing((y >= 0.5).astype(np.float32))

def validate_predictions(validation_data, validation_masks, bg_gmm, fg_gmm, lmbda, plot=True):
    """Predictions on validation set, and average results"""
    predict_count = 0
    total_score = 0
    for data, mask in zip(validation_data, validation_masks):
        predict_count += 1
        y = predict_foreground(data.T, bg_gmm, fg_gmm, lmbda=lmbda)
        score = accuracy_score(y + 1, mask.flatten().round() + 1)
        print("Prediction #{0}, accuracy:{1}%".format(predict_count, round(score *100, 2)) )
        total_score += score
        if plot:
            show_image(y.reshape(NEW_SIZE[::-1]), cmap='gray')
            plt.show()
    print("Average accuracy:{0}%".format(round(total_score/predict_count *100, 2)) )
```

```python
validate_predictions(validate_data, validate_masks, bg_gmm, fg_gmm, lmbda=0.4)
```

## 6-Fold Cross-Validation
Since the data set is limited, cross-validation might be used to evaluate the skill of the GMM for the unseen test data. A 6-Fold method is used, which splits the data into 6 groups of 8 puzzle pieces each. The method is popular since it results in a less biased estimate of the model's skill than other methods. The procedure used follows:
- Shuffle the test set data randomly
- Split the data set randomly into the 6 groups (with 8 image data each.)
- Then for each of these groups
 - Assign the group as a test set
 - Use all the remaining groups as the training set
 - Fit the model on the training set and evaluate it using the test set
 - Compute the score and discard the model
- Model performance is then the mean value of all scores

```python
def validate_predictions_cv(validation_data, validation_masks, bg_gmm, fg_gmm, lmbda, plot=True):
    predict_count = 0
    total_score = 0
    for data, mask in zip(validation_data, validation_masks):
        predict_count += 1
        y = predict_foreground(data.T, bg_gmm, fg_gmm, lmbda=lmbda)
        score = accuracy_score(y + 1, mask.flatten().round() + 1)
        print("Prediction #{0}, accuracy:{1}%".format(predict_count, round(score *100, 2)) )
        total_score += score
        if plot:
            show_image(y.reshape(NEW_SIZE[::-1]), cmap='gray')
            plt.show()
    print("Average accuracy:{0}%".format(round(total_score/predict_count *100, 2)) )

    return  round(total_score/predict_count *100, 2)
```

```python
def cross_val_manager(folds = 6, k_foreground=3, k_background=3, lmbda=0.1):
    #initialize accuracy list
    acc = []

    # copy the data
    data_imgs = DATA_IMGS.copy()
    data_msks = DATA_MSKS.copy()

    #create groups
    #create a random array containing the index of each img number in the data set
    #each row of this array will be a group
    img_indices = np.random.choice(np.arange(0, data_imgs.shape[0]), replace=False,
                                   size=(folds, int(data_imgs.shape[0]/folds)))

    #main loop that selects one group of images as the test set and the remainder as the training sets
    for i in tqdm(range(img_indices.shape[0])):
        print('Fold: {}'.format(i))
        training_indices = []
        testing_indices = img_indices[i]
        for row in range(img_indices.shape[0]):

            if row != i:
                #print('row: {}'.format(row))
                training_indices.append(list(img_indices[row]))
        training_indices = np.asarray(training_indices)
        #print('\ntest: {}'. format(testing_indices))
        #print('\ntrain: {}'.format(training_indices))

        #flatten the array
        train_indices = training_indices.flatten()

        #get training imgs and masks
        train_imgs = np.array([data_imgs[i] for i in train_indices])
        train_msks = np.array([data_msks[j] for j in train_indices])

        #extract features for training data
        x_train = np.array([data_utils.extract_features(img) for img in train_imgs])

        #extract bg and fg pixels
        x_train_bg = [data_utils.apply_mask(features, mask, invert_mask=True)
                      for features, mask in zip(x_train, train_msks)]
        x_train_fg = [data_utils.apply_mask(features, mask, invert_mask=False)
                      for features, mask in zip(x_train, train_masks)]
        x_train_bg = np.concatenate(x_train_bg, axis=1)
        x_train_fg = np.concatenate(x_train_fg, axis=1)

        #train bg_gmm
        print('\nTraining bg_gmm:')
        print('')
        bg_gmm_cv = GaussianMixtureModel(k_size=k_background)
        bg_gmm_cv.fit(x_train_bg.T)

        #train fg_gmm
        print('\nTraining fg_gmm:')
        print('')
        fg_gmm_cv = GaussianMixtureModel(k_size=k_foreground, tol= 1e-2)
        fg_gmm_cv.fit(x_train_fg.T)

        #get test imgs and masks
        test_imgs = np.array([data_imgs[i] for i in testing_indices])
        test_msks = np.array([data_msks[j] for j in testing_indices])

        #extract features for test set
        x_test = np.array([data_utils.extract_features(test_img) for test_img in test_imgs])

        #get accuracy score for test set
        test_acc = validate_predictions_cv(x_test, test_msks, bg_gmm_cv, fg_gmm_cv, lmbda = lmbda, plot=True)

        #append accuracy to acc list
        acc.append(test_acc)

    print('\nThe accuracy scores achieved for each fold:')
    print(acc)

    final_acc = sum(acc)/len(acc)
    print('\nThe final accuracy of the system: {}'.format(final_acc))
```

```python
cross_val_manager(folds=6)
```

```python

```


# Free-Body Diagram for particles

> Renato Naville Watanabe
> [Laboratory of Biomechanics and Motor Control](http://pesquisa.ufabc.edu.br/bmclab)
> Federal University of ABC, Brazil

Contents

1  Python setup

2  Free-Body Diagram

3  Steps to draw a free-body diagram (FBD)

4  Basic element and forces

4.1  Gravity
4.2  Spring
4.3  Damping

5  Examples of free-body diagram

5.1  No force acting on the particle
5.2  Gravity force acting on the particle
5.3  Ground reaction force
5.4  Mass-spring system with horizontal movement
5.5  Linear spring in bidimensional movement at horizontal plane
5.6  Particle under action of gravity and linear air resistance
5.7  Particle under action of gravity and nonlinear air resistance
5.8  Linear spring and damping on bidimensional horizontal movement
5.9  Simple muscle model

6  Further reading

7  Video lectures on the internet

8  Problems

9  References

## Python setup

```python
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
%matplotlib inline
sns.set_context('notebook', font_scale=1.2)
```

## Free-Body Diagram

In the mechanical modeling of an inanimate or living system, composed by one or more bodies (bodies as units that are mechanically isolated according to the question one is trying to answer), it is convenient to isolate each body (be they originally interconnected or not) and identify each force and moment of force (torque) that act on this body in order to apply the laws of mechanics.

**The free body diagram (FBD) of a mechanical system or model is the representation in a diagram of all forces and moments of force acting on each body, isolated from the rest of the system.**

The term free means that each body, which maybe was part of a connected system, is represented as isolated (free) and any existent contact force is represented in the diagram as forces (action and reaction) acting on the formerly connected bodies. Then, the laws of mechanics are applied on each body, and the unknown movement, force or moment of force can be found if the system of equations is determined (the number of unknown variables can not be greater than the number of equations for each body).

How exactly a FBD is drawn for a mechanical model of something is dependent on what one is trying to find. For example, the air resistance might be neglected or not when modeling the movement of an object and the number of parts the system is divided is dependent on what is needed to know about the model.

The use of FBD is very common in biomechanics; a typical use is to use the FBD in order to determine the forces and torques on the ankle, knee, and hip joints of the lower limb (foot, leg, and thigh) during locomotion, and the FBD can be applied to any problem where the laws of mechanics are needed to solve a problem.

For now, let's study how to draw free-body diagrams for systems that can be modeled as particles.

### Steps to draw a free-body diagram (FBD)

1. Draw separately each object considered in the problem. How you separate depends on what questions you want to answer.
2. Identify the forces acting on each object. If you are analyzing more than one object, remember the Newton's third Law (action and reaction), and identify where the reaction of a force is being applied.
3. Draw all the identified forces, representing them as vectors. The vectors should be represented with the origin in the object. In the case of particles, the origin should be in the center of the particle.
4. If necessary, you should represent the reference frame in the free-body diagram.
5. After this, you can solve the problem using the Newton's second Law (see, e.g, [Newton's Laws](https://nbviewer.jupyter.org/github/BMClab/bmc/blob/master/Notebooks/newtonLawForParticles.ipynb)) to find the motion of the particle.

## Basic element and forces

### Gravity

The gravity force acts on two masses, each one atracting each other:

\begin{equation}
\vec{{\bf{F}}} = - G\frac{m_1m_2}{||\vec{\bf{r}}||^2}\frac{\vec{\bf{r}}}{||\vec{\bf{r}}||}
\end{equation}

where $G = 6.67.10^{−11} Nm^2/kg^2$ and $\vec{\bf{r}}$ is a vector with length equal to the distance between the masses and directing towards the other mass. Note the forces acting on each mass have the same absolute value.

Since the mass of the Earth is $m_1=5.9736×10^{24}kg$ and its radius is 6.371×10$^6$ m, the gravity force near the surface of the Earth is:

\begin{equation}
\vec{{\bf{F}}} = m\vec{\bf{g}}
\end{equation}

with the absolute value of $\vec{\bf{g}}$ approximately equal to 9.81 $m/s^2$, pointing towards the center of Earth.

### Spring

Spring is an element used to represent a force proportional to some length or displacement. It produces a force in the same direction of the vector linking the spring extremities and opposite to its length or displacement from an equilibrium length. Frequently it has a linear relation, but it could be nonlinear as well. The force exerted by the spring in one of the extremities is:

\begin{equation}
\vec{\bf{F}} = - k(||\vec{\bf{r}}||-l_0)\frac{\vec{\bf{r}}}{||\vec{\bf{r}}||} = -k\vec{\bf{r}} +kl_0\frac{\vec{\bf{r}}}{||\vec{\bf{r}}||} = -k\left(1-\frac{l_0}{||\vec{\bf{r}}||}\right)\vec{\bf{r}}
\end{equation}

where $\vec{\bf{r}}$ is the vector linking the extremity applying the force to the other extremity and $l_0$ is the equilibrium length of the spring.

Since the spring element is a massless element, the force in  both extremities have the same absolute value and opposite directions.

### Damping

Damper is an element used to represent a force proportional to the velocity of displacement. It produces a force in the opposite direction of its velocity.

Frequently it has a linear relation, but it could be nonlinear as well. The force exerted by the damper element in one of its extremities is:

\begin{equation}
\vec{\bf{F}} = - b||\vec{\bf{v}}||\frac{\vec{\bf{v}}}{||\vec{\bf{v}}||} = -b\vec{\bf{v}} = -b\frac{d\vec{\bf{r}}}{dt}
\end{equation}

where $\vec{\bf{r}}$ is the vector linking the extremity applying the force to the other extremity.

Since the damper element is a massless element, , the force in  both extremities have the same absolute value and opposite directions.

## Examples of free-body diagram

Let's see some examples on how to draw the free-body diagram and obtain the motion equations to solve the problems.

### 1. No force acting on the particle

The most trivial situation is a particle with no force acting on it.

The free-body diagram is below, with no force vectors acting on the particle.

Figure. Free-body diagram of a ball with no force acting on it.

In this situation, the resultant force is:

\begin{equation}
\vec{\bf{F}} = 0
\end{equation}

And the second Newton law for this particle is:

\begin{equation}
m\frac{d^2\vec{\bf{r}}}{dt^2} = 0 \quad \rightarrow \quad \frac{d^2\vec{\bf{r}}}{dt^2} = 0
\end{equation}

The motion of of the particle can be found by integrating twice both times, getting the following:

\begin{equation}
\vec{\bf{r}} = \vec{\bf{v}}_0t + \vec{\bf{r}}_0
\end{equation}

The particle continues to change its position with the same velocity it was at the beginning of the analysis. This could be predicted by Newton's first law.

### 2. Gravity force acting on the particle

Now, let's consider a ball with the gravity force acting on it. The free-body diagram is depicted below.

Figure. Free-body diagram of a ball under the influence of gravity.

The only force acting on the ball is the gravitational force:

\begin{equation}
\vec{\bf{F}}_g = - mg \; \hat{\bf{j}}
\end{equation}

Applying Newton's second Law:

\begin{equation}
\vec{\bf{F}}_g = m \frac{d^2\vec{\bf{r}}}{dt^2} \rightarrow - mg \; \hat{\bf{j}} = m \frac{d^2\vec{\bf{r}}}{dt^2} \rightarrow - g \; \hat{\bf{j}} = \frac{d^2\vec{\bf{r}}}{dt^2}
\end{equation}

Now, we can separate the equation in two components (x and y):

\begin{equation}
0 = \frac{d^2x}{dt^2}
\end{equation}

and

\begin{equation}
- g = \frac{d^2y}{dt^2}
\end{equation}

These equations were solved in [this Notebook about the Newton's laws](https://nbviewer.jupyter.org/github/BMClab/BMC/blob/master/notebooks/newtonLawForParticles.ipynb).

### 3. Ground reaction force

Now, we will analyze the situation of a particle at rest in contact with the ground. To simplify the analysis, only the vertical movement will be considered.

Figure. Free-body diagram of a ball at rest in contact with the ground.

The forces acting on the particle are the ground reaction force (often called as normal force) and the gravity force. The free-body diagram of the particle is below:

Figure. Free-body diagram of a ball under the influence of gravity.

So, the resultant force in the particle is:

\begin{equation}
    \vec{\bf{F}} = \overrightarrow{\bf{GRF}} + m\vec{\bf{g}} = \overrightarrow{\bf{GRF}} - mg \; \hat{\bf{j}}
\end{equation}

Considering only the y direction:

\begin{equation}
    F = GRF - mg
\end{equation}

Applying Newton's second law to the particle:

\begin{equation}
    m \frac{d^2y}{dt^2} = GRF - mg
\end{equation}

Note that since we have no information about how the force GRF varies along time, we cannot solve this equation. To find the position of the particle along time, one would have to measure the ground reaction force. See [the notebook on Vertical jump](http://nbviewer.jupyter.org/github/BMClab/BMC/blob/master/notebooks/VerticalJump.ipynb) for an application of this model.

### 4. Mass-spring system with horizontal movement

The example below represents a mass attached to a spring and the other extremity of the spring is fixed.

Figure. Mass-spring system with horizontal movement.

The only force force acting on the mass is from the spring. Below is the free-body diagram from the mass.

Figure. Free-body diagram of a mass-spring system.

Since the movement is horizontal, we can neglect the gravity force.

\begin{equation}
    \vec{\bf{F}} = -k\left(1-\frac{l_0}{||\vec{\bf{r}}||}\right)\vec{\bf{r}}
\end{equation}

Applying Newton's second law to the mass:

\begin{equation}
    m\frac{d^2\vec{\bf{r}}}{dt^2} = -k\left(1-\frac{l_0}{||\vec{\bf{r}}||}\right)\vec{\bf{r}} \rightarrow \frac{d^2\vec{\bf{r}}}{dt^2} = -\frac{k}{m}\left(1-\frac{l_0}{||\vec{\bf{r}}||}\right)\vec{\bf{r}}
\end{equation}

Since the movement is unidimensional, we can deal with it scalarly:

\begin{equation}
    \frac{d^2x}{dt^2} = -\frac{k}{m}\left(1-\frac{l_0}{x}\right)x = -\frac{k}{m}(x-l_0)
\end{equation}

To solve this equation numerically, we must break the equations into two first-order differential equation:

\begin{equation}
    \frac{dv_x}{dt} =  -\frac{k}{m}(x-l_0)
\end{equation}

\begin{equation}
    \frac{dx}{dt} =  v_x
\end{equation}

In the numerical solution below, we will use $k = 40 N/m$, $m = 2 kg$, $l_0 = 0.5 m$ and the mass starts from the position $x = 0.8m$ and at rest.

```python
k = 40
m = 2
l0 = 0.5
x0 = 0.8
v0 = 0

x = x0
v = v0

dt = 0.001
t = np.arange(0, 3, dt)

r = np.array([x])

for i in t[1:]:
    dxdt = v
    dvxdt = -k/m*(x-l0)
    x = x + dt*dxdt
    v = v + dt*dvxdt
    r = np.vstack((r,np.array([x])))

plt.figure(figsize=(8, 4))
plt.plot(t, r, lw=4)
plt.xlabel('t(s)')
plt.ylabel('x(m)')
plt.title('Spring displacement')
plt.show()
```

### 5. Linear spring in bidimensional movement at horizontal plane

This example below represents a system with two masses attached to a spring.
To solve the motion of both masses, we have to draw a free-body diagram for each one of the masses.

Figure. Linear spring in bidimensional movement at horizontal plane.

The only force acting on each mass is the force due to the spring. Since the movement is happening at the horizontal plane, the gravity force can be neglected.

Figure. FBD of linear spring in bidimensional movement at horizontal plane.

So, the forces acting on mass 1 is:

\begin{equation}
    \vec{\bf{F_1}} = k\left(||\vec{\bf{r_2}}-\vec{\bf{r_1}}||-l_0\right)\frac{(\vec{\bf{r_2}}-\vec{\bf{r_1}})}{||\vec{\bf{r_2}}-\vec{\bf{r_1}}||}
\end{equation}

and the forces acting on mass 2 is:

\begin{equation}
    \vec{\bf{F_2}} =k\left(||\vec{\bf{r_2}}-\vec{\bf{r_1}}||-l_0\right)\frac{(\vec{\bf{r_1}}-\vec{\bf{r_2}})}{||\vec{\bf{r_2}}-\vec{\bf{r_1}}||}
\end{equation}

Applying Newton's second law for the masses:

\begin{equation}
     m_1\frac{d^2\vec{\bf{r_1}}}{dt^2} = k\left(||\vec{\bf{r_2}}-\vec{\bf{r_1}}||-l_0\right)\frac{(\vec{\bf{r_2}}-\vec{\bf{r_1}})}{||\vec{\bf{r_2}}-\vec{\bf{r_1}}||}
    \\
    \frac{d^2\vec{\bf{r_1}}}{dt^2} = -\frac{k}{m_1}\left(1-\frac{l_0}{||\vec{\bf{r_2}}-\vec{\bf{r_1}}||}\right)\vec{\bf{r_1}}+\frac{k}{m_1}\left(1-\frac{l_0}{||\vec{\bf{r_2}}-\vec{\bf{r_1}}||}\right)\vec{\bf{r_2}}
    \\
    \frac{d^2x_1\hat{\bf{i}}}{dt^2}+\frac{d^2y_1\hat{\bf{j}}}{dt^2} = -\frac{k}{m_1}\left(1-\frac{l_0}{||\vec{\bf{r_2}}-\vec{\bf{r_1}}||}\right)(x_1\hat{\bf{i}}+y_1\hat{\bf{j}})+\frac{k}{m_1}\left(1-\frac{l_0}{||\vec{\bf{r_2}}-\vec{\bf{r_1}}||}\right)(x_2\hat{\bf{i}}+y_2\hat{\bf{j}})
\end{equation}

\begin{equation}
     m_2\frac{d^2\vec{\bf{r_2}}}{dt^2} = k\left(||\vec{\bf{r_2}}-\vec{\bf{r_1}}||-l_0\right)\frac{(\vec{\bf{r_1}}-\vec{\bf{r_2}})}{||\vec{\bf{r_2}}-\vec{\bf{r_1}}||}
    \\
    \frac{d^2\vec{\bf{r_2}}}{dt^2} = -\frac{k}{m_2}\left(1-\frac{l_0}{||\vec{\bf{r_2}}-\vec{\bf{r_1}}||}\right)\vec{\bf{r_2}}+\frac{k}{m_2}\left(1-\frac{l_0}{||\vec{\bf{r_2}}-\vec{\bf{r_1}}||}\right)\vec{\bf{r_1}}
    \\
    \frac{d^2x_2\hat{\bf{i}}}{dt^2}+\frac{d^2y_2\hat{\bf{j}}}{dt^2} = -\frac{k}{m_2}\left(1-\frac{l_0}{||\vec{\bf{r_2}}-\vec{\bf{r_1}}||}\right)(x_2\hat{\bf{i}}+y_2\hat{\bf{j}})+\frac{k}{m_2}\left(1-\frac{l_0}{||\vec{\bf{r_2}}-\vec{\bf{r_1}}||}\right)(x_1\hat{\bf{i}}+y_1\hat{\bf{j}})
\end{equation}

Now, we can separate the equations for each of the coordinates:

\begin{equation}
    \frac{d^2x_1}{dt^2} = -\frac{k}{m_1}\left(1-\frac{l_0}{||\vec{\bf{r_2}}-\vec{\bf{r_1}}||}\right)x_1+\frac{k}{m_1}\left(1-\frac{l_0}{||\vec{\bf{r_2}}-\vec{\bf{r_1}}||}\right)x_2=-\frac{k}{m_1}\left(1-\frac{l_0}{\sqrt{(x_2-x_1)^2+(y_2-y_1)^2}}\right)(x_1-x_2)
\end{equation}

\begin{equation}
    \frac{d^2y_1}{dt^2} = -\frac{k}{m_1}\left(1-\frac{l_0}{||\vec{\bf{r_2}}-\vec{\bf{r_1}}||}\right)y_1+\frac{k}{m_1}\left(1-\frac{l_0}{||\vec{\bf{r_2}}-\vec{\bf{r_1}}||}\right)y_2=-\frac{k}{m_1}\left(1-\frac{l_0}{\sqrt{(x_2-x_1)^2+(y_2-y_1)^2}}\right)(y_1-y_2)
\end{equation}

\begin{equation}
    \frac{d^2x_2}{dt^2} = -\frac{k}{m_2}\left(1-\frac{l_0}{||\vec{\bf{r_2}}-\vec{\bf{r_1}}||}\right)x_2+\frac{k}{m_2}\left(1-\frac{l_0}{||\vec{\bf{r_2}}-\vec{\bf{r_1}}||}\right)x_1=-\frac{k}{m_2}\left(1-\frac{l_0}{\sqrt{(x_2-x_1)^2+(y_2-y_1)^2}}\right)(x_2-x_1)
\end{equation}

\begin{equation}
    \frac{d^2y_2}{dt^2} = -\frac{k}{m_2}\left(1-\frac{l_0}{||\vec{\bf{r_2}}-\vec{\bf{r_1}}||}\right)y_2+\frac{k}{m_2}\left(1-\frac{l_0}{||\vec{\bf{r_2}}-\vec{\bf{r_1}}||}\right)y_1=-\frac{k}{m_2}\left(1-\frac{l_0}{\sqrt{(x_2-x_1)^2+(y_2-y_1)^2}}\right)(y_2-y_1)
\end{equation}

To solve these equations numerically, you must break these equations into first-order equations:

\begin{equation}
    \frac{dv_{x_1}}{dt} = -\frac{k}{m_1}\left(1-\frac{l_0}{\sqrt{(x_2-x_1)^2+(y_2-y_1)^2}}\right)(x_1-x_2)
\end{equation}

\begin{equation}
    \frac{dv_{y_1}}{dt} = -\frac{k}{m_1}\left(1-\frac{l_0}{\sqrt{(x_2-x_1)^2+(y_2-y_1)^2}}\right)(y_1-y_2)
\end{equation}

\begin{equation}
    \frac{dv_{x_2}}{dt} = -\frac{k}{m_2}\left(1-\frac{l_0}{\sqrt{(x_2-x_1)^2+(y_2-y_1)^2}}\right)(x_2-x_1)
\end{equation}

\begin{equation}
    \frac{dv_{y_2}}{dt} = -\frac{k}{m_2}\left(1-\frac{l_0}{\sqrt{(x_2-x_1)^2+(y_2-y_1)^2}}\right)(y_2-y_1)
\end{equation}

\begin{equation}
    \frac{dx_1}{dt} = v_{x_1}
\end{equation}

\begin{equation}
    \frac{dy_1}{dt} = v_{y_1}
\end{equation}

\begin{equation}
    \frac{dx_2}{dt} = v_{x_2}
\end{equation}

\begin{equation}
    \frac{dy_2}{dt} = v_{y_2}
\end{equation}

Note that if you did not want to know the details about the motion of each mass, but only the motion of the center of mass of the masses-spring system, you could have modeled the whole system as a single particle.

To solve the equations numerically, we will use the $m_1=1 kg$, $m_2 = 2 kg$, $l_0 = 0.5 m$, $k = 90 N/m$ and $x_{1_0} = 0 m$, $x_{2_0} = 0 m$, $y_{1_0} = 1 m$, $y_{2_0} = -1 m$, $v_{x1_0} = -2 m/s$, $v_{x2_0} = 0 m/s$, $v_{y1_0} = 0 m/s$, $v_{y2_0} = 0 m/s$.

```python
x01 = 0
y01= 0.5
x02 = 0
y02 = -0.5
vx01 = 0.1
vy01 = 0
vx02 = -0.1
vy02 = 0

x1= x01
y1 = y01
x2= x02
y2 = y02
vx1= vx01
vy1 = vy01
vx2= vx02
vy2 = vy02
r1 = np.array([x1,y1])
r2 = np.array([x2,y2])

k = 30
m1 = 1
m2 = 1
l0 = 0.5

dt = 0.0001
t = np.arange(0,5,dt)

for i in t[1:]:

    dvx1dt = -k/m1*(x1-x2)*(1-l0/np.sqrt((x2-x1)**2+(y2-y1)**2))
    dvx2dt = -k/m2*(x2-x1)*(1-l0/np.sqrt((x2-x1)**2+(y2-y1)**2))
    dvy1dt = -k/m1*(y1-y2)*(1-l0/np.sqrt((x2-x1)**2+(y2-y1)**2))
    dvy2dt = -k/m2*(y2-y1)*(1-l0/np.sqrt((x2-x1)**2+(y2-y1)**2))

    dx1dt = vx1
    dx2dt = vx2
    dy1dt = vy1
    dy2dt = vy2

    x1 = x1 + dt*dx1dt
    x2 = x2 + dt*dx2dt
    y1 = y1 + dt*dy1dt
    y2 = y2 + dt*dy2dt

    vx1 = vx1 + dt*dvx1dt
    vx2 = vx2 + dt*dvx2dt
    vy1 = vy1 + dt*dvy1dt
    vy2 = vy2 + dt*dvy2dt

    r1 = np.vstack((r1,np.array([x1,y1])))
    r2 = np.vstack((r2,np.array([x2,y2])))

springLength = np.sqrt((r1[:,0]-r2[:,0])**2+(r1[:,1]-r2[:,1])**2)
plt.figure(figsize=(8, 4))
plt.plot(t, springLength, lw=4)
plt.xlabel('t(s)')
plt.ylabel('Spring length (m)')
plt.show()

plt.figure(figsize=(8, 4))
plt.plot(r1[:,0], r1[:,1], 'r.', lw=4)
plt.plot(r2[:,0], r2[:,1], 'b.', lw=4)
plt.plot((m1*r1[:,0]+m2*r2[:,0])/(m1+m2), (m1*r1[:,1]+m2*r2[:,1])/(m1+m2),'g.')
plt.xlim(-0.7,0.7)
plt.ylim(-0.7,0.7)
plt.xlabel('x(m)')
plt.ylabel('y(m)')
plt.title('Masses position')
plt.legend(('Mass1','Mass 2','Masses center of mass'))
plt.show()
```

### 6. Particle under action of gravity and linear air resistance

Below is the free-body diagram of a particle with the gravity force and a linear drag force due to the air resistance.

Figure. Particle under action of gravity and linear air resistance.

the forces being applied in the ball are:

\begin{equation}
\vec{\bf{F}} = -mg \hat{\bf{j}} - b\vec{\bf{v}} = -mg \hat{\bf{j}} - b\frac{d\vec{\bf{r}}}{dt} = -mg \hat{\bf{j}} - b\left(\frac{dx}{dt}\hat{\bf{i}}+\frac{dy}{dt}\hat{\bf{j}}\right) = - b\frac{dx}{dt}\hat{\bf{i}} - \left(mg + b\frac{dy}{dt}\right)\hat{\bf{j}}
\end{equation}

Writing down Newton's second law:

\begin{equation}
\vec{\bf{F}} = m \frac{d^2\vec{\bf{r}}}{dt^2} \rightarrow - b\frac{dx}{dt}\hat{\bf{i}} - \left(mg + b\frac{dy}{dt}\right)\hat{\bf{j}} = m\left(\frac{d^2x}{dt^2}\hat{\bf{i}}+\frac{d^2y}{dt^2}\hat{\bf{j}}\right)
\end{equation}

Now, we can separate into one equation for each coordinate:

\begin{equation}
- b\frac{dx}{dt} = m\frac{d^2x}{dt^2} -\rightarrow \frac{d^2x}{dt^2} = -\frac{b}{m} \frac{dx}{dt}
\end{equation}

\begin{equation}
-mg - b\frac{dy}{dt} = m\frac{d^2y}{dt^2} \rightarrow \frac{d^2y}{dt^2} = -\frac{b}{m}\frac{dy}{dt} - g
\end{equation}

These equations were solved in [this notebook](https://nbviewer.jupyter.org/github/BMClab/BMC/blob/master/notebooks/newtonLawForParticles.ipynb).

### 7. Particle under action of gravity and nonlinear air resistance

Below, is the free-body diagram of a particle with the gravity force and a drag force due to the air resistance proportional to the square of the particle velocity.

Figure. Particle under action of gravity and nonlinear air resistance.

The forces being applied in the ball are:

\begin{equation}
\vec{\bf{F}} = -mg \hat{\bf{j}} - bv^2\hat{\bf{e_t}} = -mg \hat{\bf{j}} - b (v_x^2+v_y^2) \frac{v_x\hat{\bf{i}}+v_y\hat{\bf{j}}}{\sqrt{v_x^2+v_y^2}} =   -mg \hat{\bf{j}} - b \sqrt{v_x^2+v_y^2} \,(v_x\hat{\bf{i}}+v_y\hat{\bf{j}}) =  -mg \hat{\bf{j}} - b \sqrt{\left(\frac{dx}{dt} \right)^2+\left(\frac{dy}{dt} \right)^2} \,\left(\frac{dx}{dt} \hat{\bf{i}}+\frac{dy}{dt}\hat{\bf{j}}\right)
\end{equation}

Writing down Newton's second law:

\begin{equation}
\vec{\bf{F}} = m \frac{d^2\vec{\bf{r}}}{dt^2} \rightarrow -mg \hat{\bf{j}} - b \sqrt{\left(\frac{dx}{dt} \right)^2+\left(\frac{dy}{dt} \right)^2} \,\left(\frac{dx}{dt} \hat{\bf{i}}+\frac{dy}{dt}\hat{\bf{j}}\right) = m\left(\frac{d^2x}{dt^2}\hat{\bf{i}}+\frac{d^2y}{dt^2}\hat{\bf{j}}\right)
\end{equation}

Now, we can separate into one equation for each coordinate:

\begin{equation}
- b \sqrt{\left(\frac{dx}{dt} \right)^2+\left(\frac{dy}{dt} \right)^2} \,\frac{dx}{dt} = m\frac{d^2x}{dt^2} \rightarrow \frac{d^2x}{dt^2} = - \frac{b}{m} \sqrt{\left(\frac{dx}{dt} \right)^2+\left(\frac{dy}{dt} \right)^2} \,\frac{dx}{dt}
\end{equation}

\begin{equation}
-mg - b \sqrt{\left(\frac{dx}{dt} \right)^2+\left(\frac{dy}{dt} \right)^2} \,\frac{dy}{dt} = m\frac{d^2y}{dt^2} \rightarrow \frac{d^2y}{dt^2} = - \frac{b}{m} \sqrt{\left(\frac{dx}{dt} \right)^2+\left(\frac{dy}{dt} \right)^2} \,\frac{dy}{dt} -g
\end{equation}

These equations were solved numerically in [this notebook](https://nbviewer.jupyter.org/github/BMClab/BMC/blob/master/notebooks/newtonLawForParticles.ipynb).

### 8. Linear spring and damping on bidimensional horizontal movement

This situation is very similar to the example of horizontal movement with one spring and two masses, with a damper added in parallel to the spring.

Figure. Linear spring and damping on bidimensional horizontal movement.

Now, the forces acting on each mass are the force due to the spring and the force due to the damper.

Figure. FBD of linear spring and damping on bidimensional horizontal movement.

So, the forces acting on mass 1 is:

\begin{equation}
    \vec{\bf{F_1}} = b\frac{d(\vec{\bf{r_2}}-\vec{\bf{r_1}})}{dt} +  k\left(||\vec{\bf{r_2}}-\vec{\bf{r_1}}||-l_0\right)\frac{(\vec{\bf{r_2}}-\vec{\bf{r_1}})}{||\vec{\bf{r_2}}-\vec{\bf{r_1}}||} =  b\frac{d(\vec{\bf{r_2}}-\vec{\bf{r_1}})}{dt} +  k\left(1-\frac{l_0}{||\vec{\bf{r_2}}-\vec{\bf{r_1}}||}\right)(\vec{\bf{r_2}}-\vec{\bf{r_1}})
\end{equation}

and the forces acting on mass 2 is:

\begin{equation}
    \vec{\bf{F_2}} = b\frac{d(\vec{\bf{r_1}}-\vec{\bf{r_2}})}{dt} +  k\left(||\vec{\bf{r_2}}-\vec{\bf{r_1}}||-l_0\right)\frac{(\vec{\bf{r_1}}-\vec{\bf{r_2}})}{||\vec{\bf{r_1}}-\vec{\bf{r_2}}||}= b\frac{d(\vec{\bf{r_1}}-\vec{\bf{r_2}})}{dt} +  k\left(1-\frac{l_0}{||\vec{\bf{r_2}}-\vec{\bf{r_1}}||}\right)(\vec{\bf{r_1}}-\vec{\bf{r_2}})
\end{equation}

Applying the Newton's second law for the masses:

\begin{equation}
     m_1\frac{d^2\vec{\bf{r_1}}}{dt^2} = b\frac{d(\vec{\bf{r_2}}-\vec{\bf{r_1}})}{dt}+k\left(1-\frac{l_0}{||\vec{\bf{r_2}}-\vec{\bf{r_1}}||}\right)(\vec{\bf{r_2}}-\vec{\bf{r_1}})
    \end{equation}

\begin{equation}
     \frac{d^2\vec{\bf{r_1}}}{dt^2} = -\frac{b}{m_1}\frac{d\vec{\bf{r_1}}}{dt} -\frac{k}{m_1}\left(1-\frac{l_0}{||\vec{\bf{r_2}}-\vec{\bf{r_1}}||}\right)\vec{\bf{r_1}} + \frac{b}{m_1}\frac{d\vec{\bf{r_2}}}{dt}+\frac{k}{m_1}\left(1-\frac{l_0}{||\vec{\bf{r_2}}-\vec{\bf{r_1}}||}\right)\vec{\bf{r_2}}
\end{equation}

\begin{equation}
     \frac{d^2x_1\hat{\bf{i}}}{dt^2}+\frac{d^2y_1\hat{\bf{j}}}{dt^2} = -\frac{b}{m_1}\left(\frac{dx_1\hat{\bf{i}}}{dt}+\frac{dy_1\hat{\bf{j}}}{dt}\right)-\frac{k}{m_1}\left(1-\frac{l_0}{||\vec{\bf{r_2}}-\vec{\bf{r_1}}||}\right)(x_1\hat{\bf{i}}+y_1\hat{\bf{j}})+\frac{b}{m_1}\left(\frac{dx_2\hat{\bf{i}}}{dt}+\frac{dy_2\hat{\bf{j}}}{dt}\right)+\frac{k}{m_1}\left(1-\frac{l_0}{||\vec{\bf{r_2}}-\vec{\bf{r_1}}||}\right)(x_2\hat{\bf{i}}+y_2\hat{\bf{j}}) = -\frac{b}{m_1}\left(\frac{dx_1\hat{\bf{i}}}{dt}+\frac{dy_1\hat{\bf{j}}}{dt}-\frac{dx_2\hat{\bf{i}}}{dt}-\frac{dy_2\hat{\bf{j}}}{dt}\right)-\frac{k}{m_1}\left(1-\frac{l_0}{\sqrt{(x_2-x_1)^2+(y_2-y_1)^2}}\right)(x_1\hat{\bf{i}}+y_1\hat{\bf{j}}-x_2\hat{\bf{i}}-y_2\hat{\bf{j}})
\end{equation}

\begin{equation}
     m_2\frac{d^2\vec{\bf{r_2}}}{dt^2} = b\frac{d(\vec{\bf{r_1}}-\vec{\bf{r_2}})}{dt}+k\left(1-\frac{l_0}{||\vec{\bf{r_2}}-\vec{\bf{r_1}}||}\right)(\vec{\bf{r_1}}-\vec{\bf{r_2}})
\end{equation}

\begin{equation}
     \frac{d^2\vec{\bf{r_2}}}{dt^2} = -\frac{b}{m_2}\frac{d\vec{\bf{r_2}}}{dt} -\frac{k}{m_2}\left(1-\frac{l_0}{||\vec{\bf{r_2}}-\vec{\bf{r_1}}||}\right)\vec{\bf{r_2}} + \frac{b}{m_2}\frac{d\vec{\bf{r_1}}}{dt}+\frac{k}{m_2}\left(1-\frac{l_0}{||\vec{\bf{r_2}}-\vec{\bf{r_1}}||}\right)\vec{\bf{r_1}}
\end{equation}

\begin{equation}
    \frac{d^2x_2\hat{\bf{i}}}{dt^2}+\frac{d^2y_2\hat{\bf{j}}}{dt^2} = -\frac{b}{m_2}\left(\frac{dx_2\hat{\bf{i}}}{dt}+\frac{dy_2\hat{\bf{j}}}{dt}\right)-\frac{k}{m_2}\left(1-\frac{l_0}{||\vec{\bf{r_2}}-\vec{\bf{r_1}}||}\right)(x_2\hat{\bf{i}}+y_2\hat{\bf{j}})+\frac{b}{m_2}\left(\frac{dx_1\hat{\bf{i}}}{dt}+\frac{dy_1\hat{\bf{j}}}{dt}\right)+\frac{k}{m_2}\left(1-\frac{l_0}{||\vec{\bf{r_2}}-\vec{\bf{r_1}}||}\right)(x_1\hat{\bf{i}}+y_1\hat{\bf{j}})=-\frac{b}{m_2}\left(\frac{dx_2\hat{\bf{i}}}{dt}+\frac{dy_2\hat{\bf{j}}}{dt}-\frac{dx_1\hat{\bf{i}}}{dt}-\frac{dy_1\hat{\bf{j}}}{dt}\right)-\frac{k}{m_2}\left(1-\frac{l_0}{\sqrt{(x_2-x_1)^2+(y_2-y_1)^2}}\right)(x_2\hat{\bf{i}}+y_2\hat{\bf{j}}-x_1\hat{\bf{i}}-y_1\hat{\bf{j}})
\end{equation}

Now, we can separate the equations for each of the coordinates:

\begin{equation}
    \frac{d^2x_1}{dt^2} = -\frac{b}{m_1}\left(\frac{dx_1}{dt}-\frac{dx_2}{dt}\right)-\frac{k}{m_1}\left(1-\frac{l_0}{\sqrt{(x_2-x_1)^2+(y_2-y_1)^2}}\right)(x_1-x_2)
\end{equation}

\begin{equation}
    \frac{d^2y_1}{dt^2} = -\frac{b}{m_1}\left(\frac{dy_1}{dt}-\frac{dy_2}{dt}\right)-\frac{k}{m_1}\left(1-\frac{l_0}{\sqrt{(x_2-x_1)^2+(y_2-y_1)^2}}\right)(y_1-y_2)
\end{equation}

\begin{equation}
    \frac{d^2x_2}{dt^2} = -\frac{b}{m_2}\left(\frac{dx_2}{dt}-\frac{dx_1}{dt}\right)-\frac{k}{m_2}\left(1-\frac{l_0}{\sqrt{(x_2-x_1)^2+(y_2-y_1)^2}}\right)(x_2-x_1)
\end{equation}

\begin{equation}
    \frac{d^2y_2}{dt^2} = -\frac{b}{m_2}\left(\frac{dy_2}{dt}-\frac{dy_1}{dt}\right)-\frac{k}{m_2}\left(1-\frac{l_0}{\sqrt{(x_2-x_1)^2+(y_2-y_1)^2}}\right)(y_2-y_1)
\end{equation}

If you want to solve these equations numerically, you must break these equations into first-order  equations:

\begin{equation}
    \frac{dv_{x_1}}{dt} = -\frac{b}{m_1}\left(v_{x_1}-v_{x_2}\right)-\frac{k}{m_1}\left(1-\frac{l_0}{\sqrt{(x_2-x_1)^2+(y_2-y_1)^2}}\right)(x_1-x_2)
\end{equation}

\begin{equation}
    \frac{dv_{y_1}}{dt} =  -\frac{b}{m_1}\left(v_{y_1}-v_{y_2}\right)-\frac{k}{m_1}\left(1-\frac{l_0}{\sqrt{(x_2-x_1)^2+(y_2-y_1)^2}}\right)(y_1-y_2)
\end{equation}

\begin{equation}
    \frac{dv_{x_2}}{dt} = -\frac{b}{m_2}\left(v_{x_2}-v_{x_1}\right)-\frac{k}{m_2}\left(1-\frac{l_0}{\sqrt{(x_2-x_1)^2+(y_2-y_1)^2}}\right)(x_2-x_1)
\end{equation}

\begin{equation}
    \frac{dv_{y_2}}{dt} = -\frac{b}{m_2}\left(v_{y_2}-v_{y_1}\right)-\frac{k}{m_2}\left(1-\frac{l_0}{\sqrt{(x_2-x_1)^2+(y_2-y_1)^2}}\right)(y_2-y_1)
\end{equation}

\begin{equation}
    \frac{dx_1}{dt} = v_{x_1}
\end{equation}

\begin{equation}
    \frac{dy_1}{dt} = v_{y_1}
\end{equation}

\begin{equation}
    \frac{dx_2}{dt} = v_{x_2}
\end{equation}

\begin{equation}
    \frac{dy_2}{dt} = v_{y_2}
\end{equation}

To solve the equations numerically, we will use the $m_1=1 kg$, $m_2 = 2 kg$, $l_0 = 0.5 m$, $k = 10 N/m$, $b = 0.6 Ns/m$ and $x_{1_0} = 0 m$, $x_{2_0} = 0 m$, $y_{1_0} = 1 m$, $y_{2_0} = -1 m$, $v_{x1_0} = -2 m/s$, $v_{x2_0} = 1 m/s$, $v_{y1_0} = 0 m/s$, $v_{y2_0} = 0 m/s$.

```python
x01 = 0
y01= 1
x02 = 0
y02 = -1

vx01 = -2
vy01 = 0
vx02 = 1
vy02 = 0

x1= x01
y1 = y01
x2= x02
y2 = y02
vx1= vx01
vy1 = vy01
vx2= vx02
vy2 = vy02
r1 = np.array([x1,y1])
r2 = np.array([x2,y2])

k = 10
m1 = 1
m2 = 2
b = 0.6
l0 = 0.5

dt = 0.001
t = np.arange(0,5,dt)

for i in t[1:]:
    dvx1dt = -b/m1*(vx1-vx2) -k/m1*(1-l0/np.sqrt((x2-x1)**2+(y2-y1)**2))*(x1-x2)
    dvx2dt = -b/m2*(vx2-vx1) -k/m2*(1-l0/np.sqrt((x2-x1)**2+(y2-y1)**2))*(x2-x1)
    dvy1dt = -b/m1*(vy1-vy2) -k/m1*(1-l0/np.sqrt((x2-x1)**2+(y2-y1)**2))*(y1-y2)
    dvy2dt = -b/m2*(vy2-vy1) -k/m2*(1-l0/np.sqrt((x2-x1)**2+(y2-y1)**2))*(y2-y1)
    dx1dt = vx1
    dx2dt = vx2
    dy1dt = vy1
    dy2dt = vy2
    x1 = x1 + dt*dx1dt
    x2 = x2 + dt*dx2dt
    y1 = y1 + dt*dy1dt
    y2 = y2 + dt*dy2dt
    vx1 = vx1 + dt*dvx1dt
    vx2 = vx2 + dt*dvx2dt
    vy1 = vy1 + dt*dvy1dt
    vy2 = vy2 + dt*dvy2dt
    r1 = np.vstack((r1,np.array([x1,y1])))
    r2 = np.vstack((r2,np.array([x2,y2])))

springDampLength = np.sqrt((r1[:,0]-r2[:,0])**2+(r1[:,1]-r2[:,1])**2)

plt.figure(figsize=(8, 4))
plt.plot(t, springDampLength, lw=4)
plt.xlabel('t(s)')
plt.ylabel('Spring length (m)')
plt.show()

plt.figure(figsize=(8, 4))
plt.plot(r1[:,0], r1[:,1], 'r.', lw=4)
plt.plot(r2[:,0], r2[:,1], 'b.', lw=4)
plt.plot((m1*r1[:,0]+m2*r2[:,0])/(m1+m2), (m1*r1[:,1]+m2*r2[:,1])/(m1+m2),'g.')
plt.xlim(-2,2)
plt.ylim(-2,2)
plt.xlabel('x(m)')
plt.ylabel('y(m)')
plt.title('Masses position')
plt.legend(('Mass1','Mass 2','Masses center of mass'))
plt.show()
```

### 9. Simple muscle model

The diagram below shows a simple muscle model. The spring in the left represents the tendinous tissues and the spring in the right represents the elastic properties of the muscle fibers. The damping is present to model the viscous properties of the muscle fibers, the element CE is the contractile element (force production) and the mass $m$ is the muscle mass.

The length $L_{MT}$ is the length of the muscle plus the tendon. In our model $L_{MT}$ is constant, but it could be a function of the joint angle.

Figure. Simple muscle model.

The length of the tendon will be denoted by $l_T(t)$ and the muscle length, by $l_{m}(t)$. Both lengths are related by each other  by the following expression:

\begin{equation}
l_t(t) + l_m(t) = L_{MT}
\end{equation}

The free-body diagram of the muscle mass is depicted below.

Figure. FBD of simple muscle model.

The resultant force being applied in the muscle mass is:

$$\vec{\bf{F}} = -k_T(||\vec{\bf{r_m}}||-l_{t_0})\frac{\vec{\bf{r_m}}}{||\vec{\bf{r_m}}||} + b\frac{d(L_{MT}\hat{\bf{i}} - \vec{\bf{r_{m}}})}{dt} + k_m (||L_{MT}\hat{\bf{i}} - \vec{\bf{r_{m}}}||-l_{m_0})\frac{L_{MT}\hat{\bf{i}} - \vec{\bf{r_{m}}}}{||L_{MT}\hat{\bf{i}} - \vec{\bf{r_{m}}}||} +\vec{\bf{F}}{\bf{_{CE}}}(t)$$

where $\vec{\bf{r_m}}$ is the muscle mass position.

Since the model is unidimensional, we can assume that the force $\vec{\bf{F}}\bf{_{CE}}(t)$ is in the x direction, so the analysis will be done only in this direction.

$$F = -k_T(l_t-l_{t_0}) + b\frac{d(L_{MT} - l_t)}{dt} + k_m (l_m-l_{m_0}) + F_{CE}(t) \\
F = -k_T(l_t-l_{t_0}) -b\frac{dl_t}{dt} + k_m (L_{MT}-l_t-l_{m_0}) + F_{CE}(t) \\
F = -b\frac{dl_t}{dt}-(k_T+k_m)l_t+F_{CE}(t)+k_Tl_{t_0}+k_m(L_{MT}-l_{m_0})$$

Applying the Newton's second law:

$$m\frac{d^2l_t}{dt^2} = -b\frac{dl_t}{dt}-(k_T+k_m)l_t+F_{CE}(t)+k_Tl_{t_0}+k_m(L_{MT}-l_{m_0})$$

To solve this equation, we must break the equation into two first-order differential equations:

\begin{equation}
\frac{dvt}{dt} = - \frac{b}{m}v_t - \frac{k_T+k_m}{m}l_t +\frac{F_{CE}(t)}{m} + \frac{k_T}{m}l_{t_0}+\frac{k_m}{m}(L_{MT}-l_{m_0})
\end{equation}

\begin{equation}
\frac{d l_t}{dt} = v_t
\end{equation}

Now, we can solve these equations by using some numerical method. To obtain the solution, we will use the damping factor of the muscle as $b = 10\,Ns/m$, the muscle mass is $m = 2 kg$, the stiffness of the tendon as $k_t=1000\,N/m$ and the elastic element of the muscle as $k_m=1500\,N/m$. The tendon-length is $L_{MT} = 0.35\,m$, and the tendon equilibrium length is $l_{t0} = 0.28\,m$ and the muscle fiber equilibrium length is $l_{m0} = 0.07\,m$. Both the tendon and the muscle fiber are at their equilibrium lengths and at rest.

Also, we will model the force of the contractile element as a Heaviside step of $90\,N$ (90 N beginning at $t=0$), but normally it is modeled as a function of $l_m$ and $v_m$ having a neural activation signal as input.

```python
m = 2
b = 10
km = 1500
kt = 1000
lt0 = 0.28
lm0 = 0.07

Lmt = 0.35
vt0 = 0

dt = 0.0001
t = np.arange(0, 10, dt)

Fce = 90

lt = lt0
vt = vt0
ltp = np.array([lt0])
lmp = np.array([lm0])
Ft = np.array([0])
for i in range(1,len(t)):
    dvtdt = -b/m*vt-(kt+km)/m*lt  + Fce/m + kt/m*lt0 +km/m*(Lmt-lm0)
    dltdt = vt
    vt = vt + dt*dvtdt
    lt = lt + dt*dltdt
    Ft = np.vstack((Ft,np.array(kt*(lt-lt0))))
    ltp = np.vstack((ltp,np.array(lt)))
    lmp = np.vstack((lmp,np.array(Lmt - lt)))

plt.figure(figsize=(8, 4))
plt.plot(t, Ft, lw=4)
plt.xlabel('t(s)')
plt.ylabel('Tendon force (N)')
plt.show()
plt.figure(figsize=(8, 4))
plt.plot(t, ltp, lw=4)
plt.plot(t, lmp, lw=4)
plt.xlabel('t(s)')
plt.ylabel('Length (m)')
plt.legend(('Tendon length', 'Muscle fiber length'))
plt.show()
```

## Further reading

- Read the 2nd chapter of the [Ruina and Rudra's book](http://ruina.tam.cornell.edu/Book/index.html) about free-body diagrams;
- Read the 13th of the [Hibbeler's book](https://drive.google.com/file/d/1sDLluWCiBCog2C11_Iu1fjv-BtfVUxBU/view) (available in the Classroom).

## Problems

1. Solve the problems 2.3.9, 2.3.20, 11.1.6, 13.1.6 (a, b, c, d, f), 13.1.7, 13.1.10 (a, b) from Ruina and Pratap's book.
2. Check examples 13.1, 13.4 and 13.5 from Hibbeler's book.

## References

- Ruina A, Rudra P (2019) [Introduction to Statics and Dynamics](http://ruina.tam.cornell.edu/Book/index.html). Oxford University Press.
- R. C. Hibbeler (2010) [Engineering Mechanics Dynamics](https://drive.google.com/file/d/1sDLluWCiBCog2C11_Iu1fjv-BtfVUxBU/view). 12th Edition. Pearson Prentice Hall.
- Nigg & Herzog (2006) [Biomechanics of the Musculo-skeletal System](https://books.google.com.br/books?id=hOIeAQAAIAAJ&dq=editions:ISBN0470017678). 3rd Edition. Wiley.


```julia
using MomentClosure, Catalyst, Latexify, OrdinaryDiffEq
```

    ┌ Info: Precompiling MomentClosure [01a1b25a-ecf0-48c5-ae58-55bfd5393600]
    └ @ Base loading.jl:1278

```julia
@parameters t, k_on, k_off, k_p, γ_p, b
@variables p(t), g(t)

vars = [g, p]

ps = [k_on, k_off, k_p, γ_p, b]

S = [1 -1 0 0;
     0 0 b -1]

# G* -> G : 0 -> g
# G -> G* : g -> 0
# g = 1 (G) or 0 (G*)

as = [k_on*(1-g),    # 0 -> g
      k_off*g*(p^2), # g -> 0
      k_p*g,         # 0 -> mP, m ~ Geometric(mean=b)
      γ_p*p]         # p -> 0

binary_vars = [1]

rn = ReactionSystemMod(t, vars, ps, as, S)
```

    ReactionSystemMod(t, Term{Real,Nothing}[g(t), p(t)], Sym{ModelingToolkit.Parameter{Real},Nothing}[k_on, k_off, k_p, γ_p, b], SymbolicUtils.Mul{Real,Int64,Dict{Any,Number},Nothing}[k_on*(1 - g(t)), k_off*g(t)*(p(t)^2), k_p*g(t), γ_p*p(t)], Any[1 -1 0 0; 0 0 b -1])

```julia
raw_eqs = generate_raw_moment_eqs(rn, 3)
closed_raw_eqs = moment_closure(raw_eqs, "conditional derivative matching", binary_vars)
latexify(closed_raw_eqs)
```

\begin{align*}
\frac{d\mu{_{10}}}{dt} =& k_{on} - k_{off} \mu{_{12}} - k_{on} \mu{_{10}} \\
\frac{d\mu{_{01}}}{dt} =& b k_{p} \mu{_{10}} - \gamma_{p} \mu{_{01}} \\
\frac{d\mu{_{11}}}{dt} =& k_{on} \mu{_{01}} + b k_{p} \mu{_{10}} - k_{on} \mu{_{11}} - \gamma_{p} \mu{_{11}} - k_{off} \mu{_{10}} \mu{_{11}}^{-3} \mu{_{12}}^{3} \\
\frac{d\mu{_{02}}}{dt} =& \gamma_{p} \mu{_{01}} + b k_{p} \mu{_{10}} + 2 b k_{p} \mu{_{11}} + 2 k_{p} \mu{_{10}} b^{2} - 2 \gamma_{p} \mu{_{02}} \\
\frac{d\mu{_{12}}}{dt} =& k_{on} \mu{_{02}} + \gamma_{p} \mu{_{11}} + b k_{p} \mu{_{10}} + 2 b k_{p} \mu{_{11}} + 2 k_{p} \mu{_{10}} b^{2} - k_{on} \mu{_{12}} - 2 \gamma_{p} \mu{_{12}} - k_{off} \mu{_{10}}^{3} \mu{_{11}}^{-8} \mu{_{12}}^{6} \\
\frac{d\mu{_{03}}}{dt} =& b k_{p} \mu{_{10}} + 3 \gamma_{p} \mu{_{02}} + 3 b k_{p} \mu{_{11}} + 3 b k_{p} \mu{_{12}} + 6 k_{p} \mu{_{10}} b^{2} + 6 k_{p} \mu{_{10}} b^{3} + 6 k_{p} \mu{_{11}} b^{2} - \gamma_{p} \mu{_{01}} - 3 \gamma_{p} \mu{_{03}}
\end{align*}

Notation: $\mu_{03} =\langle n_2^3 \rangle$, $\mu_{12} = \langle n_1 n_2^2 \rangle$

```julia
raw_eqs.odes
```

\begin{align}
\frac{d\mu{_{10}}(t)}{dt} =& k_{on} -1.0 k_{off} \mathrm{\mu{_{12}}}\left( t \right) -1 k_{on} \mathrm{\mu{_{10}}}\left( t \right) \\
\frac{d\mu{_{01}}(t)}{dt} =& b k_{p} \mathrm{\mu{_{10}}}\left( t \right) -1.0 \gamma_{p} \mathrm{\mu{_{01}}}\left( t \right) \\
\frac{d\mu{_{20}}(t)}{dt} =& k_{on} + k_{off} \mathrm{\mu{_{12}}}\left( t \right) + k_{on} \mathrm{\mu{_{10}}}\left( t \right) -2.0 k_{off} \mathrm{\mu{_{22}}}\left( t \right) -2.0 k_{on} \mathrm{\mu{_{20}}}\left( t \right) \\
\frac{d\mu{_{11}}(t)}{dt} =& k_{on} \mathrm{\mu{_{01}}}\left( t \right) + b k_{p} \mathrm{\mu{_{20}}}\left( t \right) -1.0 k_{off} \mathrm{\mu{_{13}}}\left( t \right) -1 k_{on} \mathrm{\mu{_{11}}}\left( t \right) -1.0 \gamma_{p} \mathrm{\mu{_{11}}}\left( t \right) \\
\frac{d\mu{_{02}}(t)}{dt} =& \gamma_{p} \mathrm{\mu{_{01}}}\left( t \right) + b k_{p} \mathrm{\mu{_{10}}}\left( t \right) -2.0 \gamma_{p} \mathrm{\mu{_{02}}}\left( t \right) + 2 b k_{p} \mathrm{\mu{_{11}}}\left( t \right) + 2 k_{p} \mathrm{\mu{_{10}}}\left( t \right) b^{2} \\
\frac{d\mu{_{30}}(t)}{dt} =& k_{on} -1.0 k_{off} \mathrm{\mu{_{12}}}\left( t \right) + 3.0 k_{off} \mathrm{\mu{_{22}}}\left( t \right) -3.0 k_{off} \mathrm{\mu{_{32}}}\left( t \right) + 2.0 k_{on} \mathrm{\mu{_{10}}}\left( t \right) -3.0 k_{on} \mathrm{\mu{_{30}}}\left( t \right) \\
\frac{d\mu{_{21}}(t)}{dt} =& k_{off} \mathrm{\mu{_{13}}}\left( t \right) + k_{on} \mathrm{\mu{_{01}}}\left( t \right) + k_{on} \mathrm{\mu{_{11}}}\left( t \right) + b k_{p} \mathrm{\mu{_{30}}}\left( t \right) -2.0 k_{off} \mathrm{\mu{_{23}}}\left( t \right) -2.0 k_{on} \mathrm{\mu{_{21}}}\left( t \right) -1.0 \gamma_{p} \mathrm{\mu{_{21}}}\left( t \right) \\
\frac{d\mu{_{12}}(t)}{dt} =& k_{on} \mathrm{\mu{_{02}}}\left( t \right) + \gamma_{p} \mathrm{\mu{_{11}}}\left( t \right) + b k_{p} \mathrm{\mu{_{20}}}\left( t \right) -1.0 k_{off} \mathrm{\mu{_{14}}}\left( t \right) -1 k_{on} \mathrm{\mu{_{12}}}\left( t \right) -2.0 \gamma_{p} \mathrm{\mu{_{12}}}\left( t \right) + 2 b k_{p} \mathrm{\mu{_{21}}}\left( t \right) + 2 k_{p} \mathrm{\mu{_{20}}}\left( t \right) b^{2} \\
\frac{d\mu{_{03}}(t)}{dt} =& b k_{p} \mathrm{\mu{_{10}}}\left( t \right) -1.0 \gamma_{p} \mathrm{\mu{_{01}}}\left( t \right) + 3.0 \gamma_{p} \mathrm{\mu{_{02}}}\left( t \right) -3.0 \gamma_{p} \mathrm{\mu{_{03}}}\left( t \right) + 3 b k_{p} \mathrm{\mu{_{11}}}\left( t \right) + 3 b k_{p} \mathrm{\mu{_{12}}}\left( t \right) + 6 k_{p} \mathrm{\mu{_{10}}}\left( t \right) b^{2} + 6 k_{p} \mathrm{\mu{_{10}}}\left( t \right) b^{3} + 6 k_{p} \mathrm{\mu{_{11}}}\left( t \right) b^{2}
\end{align}

```julia
latexify(closed_raw_eqs, :closure)
```

\begin{align*}
\mu{_{13}} =& \mu{_{10}} \mu{_{11}}^{-3} \mu{_{12}}^{3} \\
\mu{_{04}} =& \mu{_{01}}^{4} \mu{_{02}}^{-6} \mu{_{03}}^{4} \\
\mu{_{14}} =& \mu{_{10}}^{-1} \mu{_{11}}^{4} \mu{_{12}}^{-6} \mu{_{13}}^{4} \\
\mu{_{05}} =& \mu{_{01}}^{-5} \mu{_{02}}^{10} \mu{_{03}}^{-10} \mu{_{04}}^{5}
\end{align*}

```julia
# PARAMETER INITIALISATION
mean_p = 200
mean_b = 70
γ_p_val = 1
k_off_val = 0.001
k_on_val = 0.05

k_p_val = mean_p * γ_p_val * (k_off_val * mean_p^2 + k_on_val) / (k_on_val * mean_b)

pmap = [k_on => k_on_val,
        k_off => k_off_val,
        k_p => k_p_val,
        γ_p => γ_p_val,
        b => mean_b]

# DETERMINISTIC INITIAL CONDITIONS

μ₀ = [1, 1]
u₀map = deterministic_IC(μ₀, closed_raw_eqs)

# time interval to solve on
tspan = (0., 6.0)
dt = 0.01

@time oprob = ODEProblem(closed_raw_eqs, u₀map, tspan, pmap);
@time sol_raw = solve(oprob, Tsit5(), saveat=dt);
```

     12.381411 seconds (17.45 M allocations: 913.442 MiB, 2.98% gc time)
     11.230540 seconds (23.40 M allocations: 1.239 GiB, 12.80% gc time)

```julia
central_eqs = generate_central_moment_eqs(rn, 3)
closed_central_eqs = moment_closure(central_eqs, "conditional derivative matching", binary_vars)
latexify(closed_central_eqs, env=:mdtable)
```

|                                                                                $\frac{d\mu{_{10}}}{dt} = k_{on} - k_{off} M{_{12}} - k_{on} \mu{_{10}} - k_{off} M{_{02}} \mu{_{10}} - 2 k_{off} M{_{11}} \mu{_{01}} - k_{off} \mu{_{10}} \mu{_{01}}^{2}$ |
| -----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------:|
|                                                                                $\frac{d\mu{_{01}}}{dt} = b k_{p} \mu{_{10}} - \gamma_{p} \mu{_{01}}$ |
|                                                                                $\frac{dM{_{11}}}{dt} = b k_{p} \mu{_{10}} + k_{off} M{_{12}} \mu{_{01}} + k_{off} \mu{_{10}} \mu{_{01}}^{3} + k_{off} M{_{02}} \mu{_{01}} \mu{_{10}} + 2 k_{off} M{_{11}} \mu{_{01}}^{2} - k_{on} M{_{11}} - \gamma_{p} M{_{11}} - b k_{p} \mu{_{10}}^{2} - k_{off} \mu{_{10}} \left( M{_{11}} + \mu{_{01}} \mu{_{10}} \right)^{-3} \left( \mu{_{10}} \mu{_{01}}^{2} + 2 M{_{11}} \mu{_{01}} + M{_{12}} + M{_{02}} \mu{_{10}} \right)^{3}$ |
|                                                                                $\frac{dM{_{02}}}{dt} = \gamma_{p} \mu{_{01}} + b k_{p} \mu{_{10}} + 2 b k_{p} M{_{11}} + 2 k_{p} \mu{_{10}} b^{2} - 2 \gamma_{p} M{_{02}}$ |
| $\frac{dM{_{12}}}{dt} = \gamma_{p} M{_{11}} + b k_{p} \mu{_{10}} + k_{off} M{_{02}} M{_{12}} + k_{off} \mu{_{10}} M{_{02}}^{2} + 2 b k_{p} M{_{11}} + 2 k_{p} \mu{_{10}} b^{2} + 2 k_{off} M{_{02}} M{_{11}} \mu{_{01}} + 2 k_{off} \mu{_{01}} \mu{_{10}} \left( M{_{11}} + \mu{_{01}} \mu{_{10}} \right)^{-3} \left( \mu{_{10}} \mu{_{01}}^{2} + 2 M{_{11}} \mu{_{01}} + M{_{12}} + M{_{02}} \mu{_{10}} \right)^{3} - k_{on} M{_{12}} - 2 \gamma_{p} M{_{12}} - b k_{p} \mu{_{10}}^{2} - 2 k_{off} M{_{11}} \mu{_{01}}^{3} - k_{off} M{_{12}} \mu{_{01}}^{2} - k_{off} \mu{_{10}} \mu{_{01}}^{4} - 2 k_{p} b^{2} \mu{_{10}}^{2} - 4 b k_{p} M{_{11}} \mu{_{10}} - k_{off} \mu{_{10}}^{3} \left( M{_{11}} + \mu{_{01}} \mu{_{10}} \right)^{-8} \left( \mu{_{10}} \mu{_{01}}^{2} + 2 M{_{11}} \mu{_{01}} + M{_{12}} + M{_{02}} \mu{_{10}} \right)^{6}$ |
|                                                                                $\frac{dM{_{03}}}{dt} = b k_{p} \mu{_{10}} + 3 \gamma_{p} M{_{02}} + 3 b k_{p} M{_{11}} + 3 b k_{p} M{_{12}} + 6 k_{p} M{_{11}} b^{2} + 6 k_{p} \mu{_{10}} b^{2} + 6 k_{p} \mu{_{10}} b^{3} - 3 \gamma_{p} M{_{03}} - \gamma_{p} \mu{_{01}}$ |

```julia
latexify(closed_central_eqs)
```

\begin{align*}
\frac{d\mu{_{10}}}{dt} =& k_{on} - k_{off} M{_{12}} - k_{on} \mu{_{10}} - k_{off} M{_{02}} \mu{_{10}} - 2 k_{off} M{_{11}} \mu{_{01}} - k_{off} \mu{_{10}} \mu{_{01}}^{2} \\
\frac{d\mu{_{01}}}{dt} =& b k_{p} \mu{_{10}} - \gamma_{p} \mu{_{01}} \\
\frac{dM{_{11}}}{dt} =& b k_{p} \mu{_{10}} + k_{off} M{_{12}} \mu{_{01}} + k_{off} \mu{_{10}} \mu{_{01}}^{3} + k_{off} M{_{02}} \mu{_{01}} \mu{_{10}} + 2 k_{off} M{_{11}} \mu{_{01}}^{2} - k_{on} M{_{11}} - \gamma_{p} M{_{11}} - b k_{p} \mu{_{10}}^{2} - k_{off} \mu{_{10}} \left( M{_{11}} + \mu{_{01}} \mu{_{10}} \right)^{-3} \left( \mu{_{10}} \mu{_{01}}^{2} + 2 M{_{11}} \mu{_{01}} + M{_{12}} + M{_{02}} \mu{_{10}} \right)^{3} \\
\frac{dM{_{02}}}{dt} =& \gamma_{p} \mu{_{01}} + b k_{p} \mu{_{10}} + 2 b k_{p} M{_{11}} + 2 k_{p} \mu{_{10}} b^{2} - 2 \gamma_{p} M{_{02}} \\
\frac{dM{_{12}}}{dt} =& \gamma_{p} M{_{11}} + b k_{p} \mu{_{10}} + k_{off} M{_{02}} M{_{12}} + k_{off} \mu{_{10}} M{_{02}}^{2} + 2 b k_{p} M{_{11}} + 2 k_{p} \mu{_{10}} b^{2} + 2 k_{off} M{_{02}} M{_{11}} \mu{_{01}} + 2 k_{off} \mu{_{01}} \mu{_{10}} \left( M{_{11}} + \mu{_{01}} \mu{_{10}} \right)^{-3} \left( \mu{_{10}} \mu{_{01}}^{2} + 2 M{_{11}} \mu{_{01}} + M{_{12}} + M{_{02}} \mu{_{10}} \right)^{3} - k_{on} M{_{12}} - 2 \gamma_{p} M{_{12}} - b k_{p} \mu{_{10}}^{2} - 2 k_{off} M{_{11}} \mu{_{01}}^{3} - k_{off} M{_{12}} \mu{_{01}}^{2} - k_{off} \mu{_{10}} \mu{_{01}}^{4} - 2 k_{p} b^{2} \mu{_{10}}^{2} - 4 b k_{p} M{_{11}} \mu{_{10}} - k_{off} \mu{_{10}}^{3} \left( M{_{11}} + \mu{_{01}} \mu{_{10}} \right)^{-8} \left( \mu{_{10}} \mu{_{01}}^{2} + 2 M{_{11}} \mu{_{01}} + M{_{12}} + M{_{02}} \mu{_{10}} \right)^{6} \\
\frac{dM{_{03}}}{dt} =& b k_{p} \mu{_{10}} + 3 \gamma_{p} M{_{02}} + 3 b k_{p} M{_{11}} + 3 b k_{p} M{_{12}} + 6 k_{p} M{_{11}} b^{2} + 6 k_{p} \mu{_{10}} b^{2} + 6 k_{p} \mu{_{10}} b^{3} - 3 \gamma_{p} M{_{03}} - \gamma_{p} \mu{_{01}}
\end{align*}

```julia
u₀map = deterministic_IC(μ₀, closed_central_eqs)

@time oprob = ODEProblem(closed_central_eqs, u₀map, tspan, pmap);
@time sol_central = solve(oprob, Tsit5(), saveat=dt);
```

      2.385484 seconds (2.37 M allocations: 122.534 MiB, 2.47% gc time)
      1.989550 seconds (3.15 M allocations: 147.229 MiB, 1.38% gc time)

```julia
using Plots
plot(sol_central.t, [sol_raw[1,:], sol_central[1,:]],
    label  = ["Raw" "Central"],
    legend = true,
    xlabel = "Time [hr]",
    ylabel = "Mean number of proteins",
    lw=2,
    legendfontsize=8,
    xtickfontsize=10,
    ytickfontsize=10,
    dpi=100)
```

```julia
plot(sol_central.t, [sol_raw[2,:], sol_central[2,:]],
    label  = ["Raw" "Central"],
    legend = true,
    xlabel = "Time [hr]",
    ylabel = "Mean number of proteins",
    lw=2,
    legendfontsize=8,
    xtickfontsize=10,
    ytickfontsize=10,
    dpi=100)
```

```julia
raw_std = sqrt.(sol_raw[4,2:end] .- sol_raw[2,2:end].^2)
central_std = sqrt.(sol_central[4,2:end])
plot(sol_central.t[2:end], [raw_std, central_std],
    label  = ["Raw" "Central"],
    legend = true,
    xlabel = "Time [hr]",
    ylabel = "Mean number of proteins",
    lw=2,
    legendfontsize=8,
    xtickfontsize=10,
    ytickfontsize=10,
    dpi=100)
```


```
%load_ext autoreload
%autoreload 2

%matplotlib inline
```

    The autoreload extension is already loaded. To reload it, use:
      %reload_ext autoreload

```
#export
from exp.nb_06 import *
```

## ConvNet

Let's get the data and training interface from where we left in the last notebook.

```
x_train,y_train,x_valid,y_valid = get_data()

x_train,x_valid = normalize_to(x_train,x_valid)
train_ds,valid_ds = Dataset(x_train, y_train),Dataset(x_valid, y_valid)

nh,bs = 50,512
c = y_train.max().item()+1
loss_func = F.cross_entropy

data = DataBunch(*get_dls(train_ds, valid_ds, bs), c)
```

```
mnist_view = view_tfm(1,28,28)
cbfs = [Recorder,
        partial(AvgStatsCallback,accuracy),
        CudaCallback,
        partial(BatchTransformXCallback, mnist_view)]
```

```
nfs = [8,16,32,64,64]
```

```
learn,run = get_learn_run(nfs, data, 0.4, conv_layer, cbs=cbfs)
```

```
%time run.fit(2, learn)
```

    train: [1.103750625, tensor(0.6346, device='cuda:0')]
    valid: [0.20124637451171876, tensor(0.9387, device='cuda:0')]
    train: [0.17081765625, tensor(0.9476, device='cuda:0')]
    valid: [0.11752720947265625, tensor(0.9648, device='cuda:0')]
    CPU times: user 3.99 s, sys: 1.19 s, total: 5.18 s
    Wall time: 5.1 s

## Batchnorm

### Custom

Let's start by building our own `BatchNorm` layer from scratch.

```
class BatchNorm(nn.Module):
    def __init__(self, nf, mom=0.1, eps=1e-5):
        super().__init__()
        # NB: pytorch bn mom is opposite of what you'd expect
        self.mom,self.eps = mom,eps
        self.mults = nn.Parameter(torch.ones (nf,1,1))
        self.adds  = nn.Parameter(torch.zeros(nf,1,1))
        self.register_buffer('vars',  torch.ones(1,nf,1,1))
        self.register_buffer('means', torch.zeros(1,nf,1,1))

    def update_stats(self, x):
        m = x.mean((0,2,3), keepdim=True)
        v = x.var ((0,2,3), keepdim=True)
        self.means.lerp_(m, self.mom)
        self.vars.lerp_ (v, self.mom)
        return m,v

    def forward(self, x):
        if self.training:
            with torch.no_grad(): m,v = self.update_stats(x)
        else: m,v = self.means,self.vars
        x = (x-m) / (v+self.eps).sqrt()
        return x*self.mults + self.adds
```

```
def conv_layer(ni, nf, ks=3, stride=2, bn=True, **kwargs):
    # No bias needed if using bn
    layers = [nn.Conv2d(ni, nf, ks, padding=ks//2, stride=stride, bias=not bn),
              GeneralRelu(**kwargs)]
    if bn: layers.append(BatchNorm(nf))
    return nn.Sequential(*layers)
```

```
#export
def init_cnn_(m, f):
    if isinstance(m, nn.Conv2d):
        f(m.weight, a=0.1)
        if getattr(m, 'bias', None) is not None: m.bias.data.zero_()
    for l in m.children(): init_cnn_(l, f)

def init_cnn(m, uniform=False):
    f = init.kaiming_uniform_ if uniform else init.kaiming_normal_
    init_cnn_(m, f)

def get_learn_run(nfs, data, lr, layer, cbs=None, opt_func=None, uniform=False, **kwargs):
    model = get_cnn_model(data, nfs, layer, **kwargs)
    init_cnn(model, uniform=uniform)
    return get_runner(model, data, lr=lr, cbs=cbs, opt_func=opt_func)
```

We can then use it in training and see how it helps keep the activations means to 0 and the std to 1.

```
learn,run = get_learn_run(nfs, data, 1., conv_layer, cbs=cbfs)
```

```
with Hooks(learn.model, append_stats) as hooks:
    run.fit(1, learn)
    fig,(ax0,ax1) = plt.subplots(1,2, figsize=(10,4))
    for h in hooks[:-1]:
        ms,ss = h.stats
        ax0.plot(ms[:10])
        ax1.plot(ss[:10])
        h.remove()
    plt.legend(range(6));

    fig,(ax0,ax1) = plt.subplots(1,2, figsize=(10,4))
    for h in hooks[:-1]:
        ms,ss = h.stats
        ax0.plot(ms)
        ax1.plot(ss)
```

```
learn,run = get_learn_run(nfs, data, 1.0, conv_layer, cbs=cbfs)
```

```
%time run.fit(3, learn)
```

    train: [0.27833810546875, tensor(0.9105, device='cuda:0')]
    valid: [0.1912491943359375, tensor(0.9386, device='cuda:0')]
    train: [0.08977265625, tensor(0.9713, device='cuda:0')]
    valid: [0.09156090698242188, tensor(0.9716, device='cuda:0')]
    train: [0.06145498046875, tensor(0.9810, device='cuda:0')]
    valid: [0.09970919799804688, tensor(0.9707, device='cuda:0')]
    CPU times: user 3.71 s, sys: 584 ms, total: 4.29 s
    Wall time: 4.29 s

### Builtin batchnorm

```
#export
def conv_layer(ni, nf, ks=3, stride=2, bn=True, **kwargs):
    layers = [nn.Conv2d(ni, nf, ks, padding=ks//2, stride=stride, bias=not bn),
              GeneralRelu(**kwargs)]
    if bn: layers.append(nn.BatchNorm2d(nf, eps=1e-5, momentum=0.1))
    return nn.Sequential(*layers)
```

```
learn,run = get_learn_run(nfs, data, 1., conv_layer, cbs=cbfs)
```

```
%time run.fit(3, learn)
```

    train: [0.27115255859375, tensor(0.9206, device='cuda:0')]
    valid: [0.1547997314453125, tensor(0.9496, device='cuda:0')]
    train: [0.07861462890625, tensor(0.9755, device='cuda:0')]
    valid: [0.07472044067382813, tensor(0.9776, device='cuda:0')]
    train: [0.0570835498046875, tensor(0.9818, device='cuda:0')]
    valid: [0.0673232666015625, tensor(0.9813, device='cuda:0')]
    CPU times: user 3.37 s, sys: 747 ms, total: 4.12 s
    Wall time: 4.12 s

### With scheduler

Now let's add the usual warm-up/annealing.

```
sched = combine_scheds([0.3, 0.7], [sched_lin(0.6, 2.), sched_lin(2., 0.1)])
```

```
learn,run = get_learn_run(nfs, data, 0.9, conv_layer, cbs=cbfs
                          +[partial(ParamScheduler,'lr', sched)])
```

```
run.fit(8, learn)
```

    train: [0.2881914453125, tensor(0.9116, device='cuda:0')]
    valid: [0.5269224609375, tensor(0.8394, device='cuda:0')]
    train: [0.153792421875, tensor(0.9551, device='cuda:0')]
    valid: [0.462953662109375, tensor(0.8514, device='cuda:0')]
    train: [0.087637158203125, tensor(0.9736, device='cuda:0')]
    valid: [0.07029829711914062, tensor(0.9788, device='cuda:0')]
    train: [0.049282421875, tensor(0.9853, device='cuda:0')]
    valid: [0.08710025024414063, tensor(0.9740, device='cuda:0')]
    train: [0.0356986328125, tensor(0.9888, device='cuda:0')]
    valid: [0.07853966674804687, tensor(0.9773, device='cuda:0')]
    train: [0.0268300439453125, tensor(0.9918, device='cuda:0')]
    valid: [0.04807376098632812, tensor(0.9870, device='cuda:0')]
    train: [0.0219412109375, tensor(0.9939, device='cuda:0')]
    valid: [0.04363873901367187, tensor(0.9882, device='cuda:0')]
    train: [0.018501048583984374, tensor(0.9951, device='cuda:0')]
    valid: [0.04355916137695313, tensor(0.9877, device='cuda:0')]

## More norms

### Layer norm

From [the paper](https://arxiv.org/abs/1607.06450): "*batch normalization cannot be applied to online learning tasks or to extremely large distributed models where the minibatches have to be small*".

General equation for a norm layer with learnable affine:

$$y = \frac{x - \mathrm{E}[x]}{ \sqrt{\mathrm{Var}[x] + \epsilon}} * \gamma + \beta$$

The difference with BatchNorm is
1. we don't keep a moving average
2. we don't average over the batches dimension but over the hidden dimension, so it's independent of the batch size

```
class LayerNorm(ScriptModule):
    __constants__ = ['eps']
    def __init__(self, eps=1e-5):
        super().__init__()
        self.eps = eps
        self.mult = nn.Parameter(tensor(1.))
        self.add  = nn.Parameter(tensor(0.))

    def forward(self, x):
        m = x.mean((1,2,3), keepdim=True)
        v = x.std ((1,2,3), keepdim=True)
        x = (x-m) / ((v+self.eps).sqrt())
        return x*self.mult + self.add
```

```
def conv_ln(ni, nf, ks=3, stride=2, bn=True, **kwargs):
    layers = [nn.Conv2d(ni, nf, ks, padding=ks//2, stride=stride, bias=True),
              GeneralRelu(**kwargs)]
    if bn: layers.append(LayerNorm())
    return nn.Sequential(*layers)
```

```
learn,run = get_learn_run(nfs, data, 0.8, conv_ln, cbs=cbfs)
```

```
%time run.fit(3, learn)
```

    train: [nan, tensor(0.1321, device='cuda:0')]
    valid: [nan, tensor(0.0991, device='cuda:0')]
    train: [nan, tensor(0.0986, device='cuda:0')]
    valid: [nan, tensor(0.0991, device='cuda:0')]
    train: [nan, tensor(0.0986, device='cuda:0')]
    valid: [nan, tensor(0.0991, device='cuda:0')]
    CPU times: user 4.56 s, sys: 862 ms, total: 5.42 s
    Wall time: 5.42 s

*Thought experiment*: can this distinguish foggy days from sunny days (assuming you're using it before the first conv)?

### Instance norm

From [the paper](https://arxiv.org/abs/1607.08022):

The key difference between **contrast** and batch normalization is that the latter applies the normalization to a  whole batch of images instead for single ones:

\begin{equation}\label{eq:bnorm}
    y_{tijk} =  \frac{x_{tijk} - \mu_{i}}{\sqrt{\sigma_i^2 + \epsilon}},
    \quad
    \mu_i = \frac{1}{HWT}\sum_{t=1}^T\sum_{l=1}^W \sum_{m=1}^H x_{tilm},
    \quad
    \sigma_i^2 = \frac{1}{HWT}\sum_{t=1}^T\sum_{l=1}^W \sum_{m=1}^H (x_{tilm} - mu_i)^2.
\end{equation}

In order to combine the effects of instance-specific normalization and batch normalization, we propose to replace the latter by the *instance normalization* (also known as *contrast normalization*) layer:

\begin{equation}\label{eq:inorm}
    y_{tijk} =  \frac{x_{tijk} - \mu_{ti}}{\sqrt{\sigma_{ti}^2 + \epsilon}},
    \quad
    \mu_{ti} = \frac{1}{HW}\sum_{l=1}^W \sum_{m=1}^H x_{tilm},
    \quad
    \sigma_{ti}^2 = \frac{1}{HW}\sum_{l=1}^W \sum_{m=1}^H (x_{tilm} - mu_{ti})^2.
\end{equation}

```
class InstanceNorm(nn.Module):
    __constants__ = ['eps']
    def __init__(self, nf, eps=1e-0):
        super().__init__()
        self.eps = eps
        self.mults = nn.Parameter(torch.ones (nf,1,1))
        self.adds  = nn.Parameter(torch.zeros(nf,1,1))

    def forward(self, x):
        m = x.mean((2,3), keepdim=True)
        v = x.std ((2,3), keepdim=True)
        res = (x-m) / ((v+self.eps).sqrt())
        return res*self.mults + self.adds
```

```
def conv_in(ni, nf, ks=3, stride=2, bn=True, **kwargs):
    layers = [nn.Conv2d(ni, nf, ks, padding=ks//2, stride=stride, bias=True),
              GeneralRelu(**kwargs)]
    if bn: layers.append(InstanceNorm(nf))
    return nn.Sequential(*layers)
```

```
learn,run = get_learn_run(nfs, data, 0.1, conv_in, cbs=cbfs)
```

```
%time run.fit(3, learn)
```

    train: [nan, tensor(0.0986, device='cuda:0')]
    valid: [nan, tensor(0.0991, device='cuda:0')]
    train: [nan, tensor(0.0986, device='cuda:0')]
    valid: [nan, tensor(0.0991, device='cuda:0')]
    train: [nan, tensor(0.0986, device='cuda:0')]
    valid: [nan, tensor(0.0991, device='cuda:0')]
    CPU times: user 4.46 s, sys: 718 ms, total: 5.18 s
    Wall time: 5.18 s

*Question*: why can't this classify anything?

Lost in all those norms? The authors from the [group norm paper](https://arxiv.org/pdf/1803.08494.pdf) have you covered:

### Group norm

*From the PyTorch docs:*

`GroupNorm(num_groups, num_channels, eps=1e-5, affine=True)`

The input channels are separated into `num_groups` groups, each containing
``num_channels / num_groups`` channels. The mean and standard-deviation are calculated
separately over the each group. $\gamma$ and $\beta$ are learnable
per-channel affine transform parameter vectorss of size `num_channels` if
`affine` is ``True``.

This layer uses statistics computed from input data in both training and
evaluation modes.

Args:
-    num_groups (int): number of groups to separate the channels into
-    num_channels (int): number of channels expected in input
-    eps: a value added to the denominator for numerical stability. Default: 1e-5
-    affine: a boolean value that when set to ``True``, this module
        has learnable per-channel affine parameters initialized to ones (for weights)
        and zeros (for biases). Default: ``True``.

Shape:
- Input: `(N, num_channels, *)`
- Output: `(N, num_channels, *)` (same shape as input)

Examples::

    >>> input = torch.randn(20, 6, 10, 10)
    >>> # Separate 6 channels into 3 groups
    >>> m = nn.GroupNorm(3, 6)
    >>> # Separate 6 channels into 6 groups (equivalent with InstanceNorm)
    >>> m = nn.GroupNorm(6, 6)
    >>> # Put all 6 channels into a single group (equivalent with LayerNorm)
    >>> m = nn.GroupNorm(1, 6)
    >>> # Activating the module
    >>> output = m(input)

## Fix small batch sizes

### What's the problem?

When we compute the statistics (mean and std) for a BatchNorm Layer on a small batch, it is possible that we get a standard deviation very close to 0. because there aren't many samples (the variance of one thing is 0. since it's equal to its mean).

```
data = DataBunch(*get_dls(train_ds, valid_ds, 2), c)
```

```
def conv_layer(ni, nf, ks=3, stride=2, bn=True, **kwargs):
    layers = [nn.Conv2d(ni, nf, ks, padding=ks//2, stride=stride, bias=not bn),
              GeneralRelu(**kwargs)]
    if bn: layers.append(nn.BatchNorm2d(nf, eps=1e-5, momentum=0.1))
    return nn.Sequential(*layers)
```

```
learn,run = get_learn_run(nfs, data, 0.4, conv_layer, cbs=cbfs)
```

```
%time run.fit(1, learn)
```

    train: [2.35619296875, tensor(0.1649, device='cuda:0')]
    valid: [2867.7198, tensor(0.2604, device='cuda:0')]
    CPU times: user 1min 32s, sys: 835 ms, total: 1min 33s
    Wall time: 1min 33s

### Running Batch Norm

To solve this problem we introduce a Running BatchNorm that uses smoother running mean and variance for the mean and std.

```
class RunningBatchNorm(nn.Module):
    def __init__(self, nf, mom=0.1, eps=1e-5):
        super().__init__()
        self.mom,self.eps = mom,eps
        self.mults = nn.Parameter(torch.ones (nf,1,1))
        self.adds = nn.Parameter(torch.zeros(nf,1,1))
        self.register_buffer('sums', torch.zeros(1,nf,1,1))
        self.register_buffer('sqrs', torch.zeros(1,nf,1,1))
        self.register_buffer('batch', tensor(0.))
        self.register_buffer('count', tensor(0.))
        self.register_buffer('step', tensor(0.))
        self.register_buffer('dbias', tensor(0.))

    def update_stats(self, x):
        bs,nc,*_ = x.shape
        self.sums.detach_()
        self.sqrs.detach_()
        dims = (0,2,3)
        s = x.sum(dims, keepdim=True)
        ss = (x*x).sum(dims, keepdim=True)
        c = self.count.new_tensor(x.numel()/nc)
        self.mom1 = self.dbias.new_tensor(1 - (1-self.mom)**bs)
        self.sums.lerp_(s, self.mom1)
        self.sqrs.lerp_(ss, self.mom1)
        self.count.lerp_(c, self.mom1)
        self.dbias.lerp_(self.mom1, self.mom1)
        self.batch += bs
        self.step += 1

    def forward(self, x):
        if self.training: self.update_stats(x)
        sums = self.sums
        sqrs = self.sqrs
        c = self.count
        if self.step<100:
            sums = sums / self.dbias
            sqrs = sqrs / self.dbias
            c    = c    / self.dbias
        means = sums/c
        vars = (sqrs/c).sub_(means*means)
        if bool(self.batch < 20): vars.clamp_min_(0.01)
        x = (x-means).div_((vars.add_(self.eps)).sqrt())
        return x.mul_(self.mults).add_(self.adds)
```

```
def conv_rbn(ni, nf, ks=3, stride=2, bn=True, **kwargs):
    layers = [nn.Conv2d(ni, nf, ks, padding=ks//2, stride=stride, bias=not bn),
              GeneralRelu(**kwargs)]
    if bn: layers.append(RunningBatchNorm(nf))
    return nn.Sequential(*layers)
```

```
learn,run = get_learn_run(nfs, data, 0.4, conv_rbn, cbs=cbfs)
```

```
%time run.fit(1, learn)
```

    train: [0.68253984375, tensor(0.8089, device='cuda:0')]
    valid: [2.0810935546875, tensor(0.9109, device='cuda:0')]
    CPU times: user 3min 46s, sys: 1.8 s, total: 3min 48s
    Wall time: 3min 49s

This solves the small batch size issue!

### What can we do in a single epoch?

Now let's see with a decent batch size what result we can get.

```
data = DataBunch(*get_dls(train_ds, valid_ds, 32), c)
```

```
learn,run = get_learn_run(nfs, data, 0.9, conv_rbn, cbs=cbfs
                          +[partial(ParamScheduler,'lr', sched_lin(1., 0.2))])
```

```
%time run.fit(1, learn)
```

    train: [0.160296025390625, tensor(0.9506, device='cuda:0')]
    valid: [0.10237916259765625, tensor(0.9753, device='cuda:0')]
    CPU times: user 16 s, sys: 1.36 s, total: 17.3 s
    Wall time: 17.4 s

## Export

```
nb_auto_export()
```

    <IPython.core.display.Javascript object>

```

```


```python
from IPython.display import Image
```

```python
cd /Users/vallevaro/Desktop/GaiaLab/notebooks/figures
```

    /Users/vallevaro/Desktop/GaiaLab/notebooks/figures

# Gaia Nominal Scanning Law

Based on Lennart Lindegren's SAG--LL-014, SAG--LL-030, SAG--LL-030. All figures are from these papers.

```python
Image('nls_angles.png')
```

Figure 1. Definition of angles $\epsilon$, $\lambda$, $\nu$, $\xi$, and $\Omega$ in the nominal scanning law.

The nominal scanning law of Gaia is described by the following angles:
* $\epsilon$: obliquity of equator. This is a $\it{constant}$ chosen to be 23º 26' 21.448''
* $\xi$: revolving angle. At any time the z axis is at this $\it{constant}$ angle from $\vec{s}$. For Gaia, the current choice is 55º.
* $\lambda_{s}(t)$: nominal longitud of the sun in the ecliptic plane. This is chosen to have a constant speed: $\dot{\lambda}(t) = \frac{2\pi}{365}$
* $\nu(t)$: revolving phase. Gives a nearly constant precession rate.
* $\Omega(t)$: spin phase.

Frames of referenced used:
* Interial Frame: $\textbf{G} = \big[ \textbf{l},\textbf{m},\textbf{n}\big]$
* Celestial frame of reference (ort): $\textbf{N} = \big[ \textbf{l},\textbf{j},\textbf{k}\big]$
* Instrument triad frame (ort): $\textbf{Z} = \big[ \textbf{x},\textbf{y},\textbf{z}\big]$
* Ecliptic triad $\textbf{E}= \big[ \textbf{l},\textbf{j},\textbf{k}\big]$

Given by:
$$
\textbf{k} = \textbf{n}cos(\epsilon) - \textbf{m} sin(\epsilon) \\
\textbf{j} = \textbf{m}cos(\epsilon) + \textbf{n} sin(\epsilon) \\ \textbf{s}=\textbf{l}cos(\lambda_{s}(t)) + \textbf{j} sin(\lambda_{s}(t)
$$

The total inertial rotation of the telescope is therefore:
$$
\begin{equation}
\vec{\omega} = \textbf{k} \dot{\lambda_{s}} + \textbf{s} \dot{\nu} + \textbf{z} \dot{\Omega}
\end{equation}
$$

The precessional motion of the z-axis is:

$$
\begin{equation}
\dot{z} = \omega \times z= \big( k \times z\big)\dot{\lambda_{s}} + \big(s \times z\big)\dot{\nu}
\end{equation}
$$

And if we consider the rate of change of the z-axis wrt time to be a constant:
$$
\begin{equation}
\frac{dz}{dt}\frac{dt}{d\lambda_{s}} = \frac{dz}{dt} \dot{\lambda_{s}}^{-1} = S
\end{equation}$$

And keeping the condition $ \|\big( k \times z \big)\|^{2} = 1 - sin^{2}(\xi)sin^{2}(\nu)$ then the following ODE for $\nu$ is obtained:

$$
\begin{equation}
\dot{\nu} = \dot{\lambda_{s}}\frac{\sqrt{S^{2} - cos^{2}(\nu)} + cos(\xi)sin(\nu)}{sin(\xi)}
\end{equation}
$$

Taking the positive sign of the square root implying increasing $\nu$.
Then, for constant inertial spin rate about the z axis using the equation for the inertial rotation $\omega$, we obtain:
$$
\begin{equation}
\dot{\Omega} = \omega_{z} - \dot{\nu}cos\xi - \dot\lambda_{s}sin\xi sin\nu
\end{equation}
$$

To see how does the longitude and latitude of the z axis changes with time, according to Figure 1, the geometry gives the following equations in the ecliptic plane:
$$
\begin{equation}
\lambda_{z}(t) = \lambda_{s}(t) + arctan\big[ tan\xi cos\nu(t) \big]. \\
\beta_{z}(t) = arcsin \big[ sin\xi sin\nu(t) \big].
\end{equation}
$$

```python
cd /Users/vallevaro/Desktop/GaiaLab/packages/scan/geometric_scanner
```

    /Users/vallevaro/Desktop/GaiaLab/packages/scan/geometric_scanner

```python
from gaia_geometric_toymodel import*
```

```python
from geometric_plots import*
```

```python
att = Attitude()
```

```python
plot_3DZ(att, 0, 365*5, 0.2)
```

Components of gaia's attitude quaternion by 0.01 step per day and for 80 days.

```python
plot_attitude(att, 0, 80, 0.01)
```

Components of gaia's attitude quaternion by 0.01 step per day and for 1 day only.

```python
plot_attitude(att, 0, 1, 0.01)
```

```python
plot_longitud_latitude(att, 0, 365*5, 0.2)
```

```python
sky = Sky(1)
```

```python
scan = Scanner(0.3, 0.2, 0.05)
```

```python
star_finder(att, sky, scan)
```

    times intercepted: 220
    times intercepted: 220

```python
plot_observations(scan, sky)
```

```python

```

notebooks/Drafts/1-Gaia Nominal Scanning Law.ipynb