# Experiments of Toy Multi-variate Gaussian Dataset: Part I (Co-lab version)
(You can just put this file on co-lab and run it. No extra files are needed.)

```python
!nvidia-smi
```

    Fri Nov 19 06:14:35 2021
    +-----------------------------------------------------------------------------+
    | NVIDIA-SMI 495.44       Driver Version: 460.32.03    CUDA Version: 11.2     |
    |-------------------------------+----------------------+----------------------+
    | GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
    | Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
    |                               |                      |               MIG M. |
    |===============================+======================+======================|
    |   0  Tesla P100-PCIE...  Off  | 00000000:00:04.0 Off |                    0 |
    | N/A   38C    P0    27W / 250W |      0MiB / 16280MiB |      0%      Default |
    |                               |                      |                  N/A |
    +-------------------------------+----------------------+----------------------+

    +-----------------------------------------------------------------------------+
    | Processes:                                                                  |
    |  GPU   GI   CI        PID   Type   Process name                  GPU Memory |
    |        ID   ID                                                   Usage      |
    |=============================================================================|
    |  No running processes found                                                 |
    +-----------------------------------------------------------------------------+

## Import packages and set hyper-parameters

```python
import numpy as np
from tqdm import tqdm
from sklearn.manifold import TSNE
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import torchvision.models as models
from torchvision import datasets, transforms
from torch.autograd import Variable
import torch.utils.data as Data
from torch.utils.data.sampler import SubsetRandomSampler
import matplotlib.pyplot as plt
import matplotlib.tri as tri
import plotly.graph_objects as go
import random
import copy
import scipy
import warnings
warnings.filterwarnings("ignore")

def seed_torch(seed=1029):
    random.seed(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    torch.backends.cudnn.benchmark = False
    torch.backends.cudnn.deterministic = True
    torch.backends.cudnn.enabled = False
seed_torch(1)

K_CLAS = 3                          # Number of classes in the toy dataset
N_Data = int(1e5)                   # Number of samples in the dataset
TVT_SPLIT = [0.05, 0.05, 0.9]       # Split ratio between of train/valid/test dataset
N_Train = int(N_Data*TVT_SPLIT[0])  # Number of training samples
N_Valid = int(N_Data*TVT_SPLIT[1])  # Number of validation samples
N_Test = int(N_Data*TVT_SPLIT[2])   # Number of test samples

BATCH_SIZE = 32                     # Training batch size
EPOCHS = 100                         # Number of training epochs
LR = 5e-4                           # Initial learning rate
LR_MIN = 5e-4                       # Minimum learning rate in cosine scheduler
X_DIM = 30                          # Dimension of input signal x
NOISE = 6                          # Noisy level when generating the dataset

LS_EPS = (1-0.05)                          # Eps of label smoothing, label is y*eps + u*(1-eps)/K_CLAS

MU_VEC = np.random.randint(-1,2,size=(K_CLAS,X_DIM))      # mu_1,...,mu_K
MU_VEC_ALL = np.tile(MU_VEC,(N_Data,1,1))
```

## Generate the mulit-variate Gaussian Dataset

In this dataset, we have $N$ samples and each sample is a 3-tuple (${x},y,p^*)$. To get one sample, we first select the label $y=k$ following an uniform distribution over all $K$ classes. After that, we sample the input signal ${x}|_{y=k}\sim\mathcal{N}({\mu}_k,\sigma^2I)$, where $\sigma$ is the noisy level for all the samples. ${\mu}_k$ is the mean vector for all the samples in class $k$. Each ${\mu}_k$ is a 30-dim vector, in which each dimension is randomly selected from $\{-1,0,1\}$. Such a process is similar to selecting 30 different features for each class. Finally, we calculate the true Bayesian probability of this sample, i.e., $p^*(y|{x})$.

To calculate $p^*(y|{x})$, we use the fact that $p^*(y|{x})\propto p({x}|y)p(y)$. As $y$ follows an uniform distribution, we have $p^*(y|{x})=\frac{p({x}|y=k)}{\sum_{j\neq k}p({x}|y=j)}$. Following $p({x}|y=k)\sim\mathcal{N}({\mu}_k,\sigma^2I)$, we find $p^*(y|{x})$ should have a Softmax form, i.e., $p=\frac{\rm{e}^{s_k}}{\sum_{j\neq k}\rm{e}^{s_j}}$. Specifically, we have:

\begin{equation}
    p^*(y=k|{x})=\frac{\rm{e}^{s_k}}{\sum_{j\neq k}\rm{e}^{s_j}};\quad s_i = -\frac{1}{2\sigma^2}\|{x}-{\mu}_i\|^2_2
\end{equation}

```python
def sigmoid(x):
  return 1/(1+np.exp(-x))

def y_to_oht(label):
  label_oht = torch.zeros(label.shape[0],K_CLAS).to(label.device)
  label_oht.scatter_(1,label,1)
  label_oht = label_oht.float()
  return label_oht

def data_gen(x,y,p):
  '''
    Pack everything to a train_loader and a val_lodaer
  '''
  x, y, p = torch.tensor(x), torch.tensor(y), torch.tensor(p)
  dataset=Data.TensorDataset(x, y, p)
  indices = list(range(N_Data))
  np.random.shuffle(indices)
  train_indices, val_indices, test_indices = indices[:N_Train], indices[N_Train:N_Train+N_Valid], indices[N_Train+N_Valid:]
  train_sampler = SubsetRandomSampler(train_indices)
  valid_sampler = SubsetRandomSampler(val_indices)
  test_sampler = SubsetRandomSampler(test_indices)
  train_loader = Data.DataLoader(dataset, batch_size=BATCH_SIZE, sampler=train_sampler, drop_last = True)
  valid_loader = Data.DataLoader(dataset, batch_size=N_Valid, sampler=valid_sampler, drop_last = True)
  test_loader = Data.DataLoader(dataset, batch_size=N_Test, sampler=test_sampler, drop_last = True)
  return train_loader, valid_loader, test_loader

# =========== Generate all x, y and p===============================
y_true = np.random.randint(0,K_CLAS,[N_Data,1]).astype(np.float32)
mu_true = np.zeros((N_Data, X_DIM))
for i in range(N_Data):
  mu_true[i,:] = MU_VEC[y_true[i].astype(np.int),:]
x_true = mu_true + np.random.randn(N_Data, X_DIM)*np.sqrt(NOISE)

logits = np.zeros((N_Data,K_CLAS))
for k in range(K_CLAS):
  logits[:,k] = np.linalg.norm(x_true - MU_VEC_ALL[:,k,:], axis=1)**2*(-0.5/NOISE)
logits = torch.from_numpy(logits)
p_true = F.softmax(logits, 1)
```

```python
train_loader, valid_loader, test_loader = data_gen(x_true,y_true,p_true)
```

## Observe the tSNE plot and histogram of $\mathbf{p}^*$

```python
X_embedded = TSNE(n_components=2, perplexity=20).fit_transform(x_true[:3000])
```

```python
for k in range(K_CLAS):
  plt.scatter(X_embedded[y_true[:3000].squeeze()==k,0],X_embedded[y_true[:3000].squeeze()==k,1],label='Class'+str(k),alpha=0.3)
plt.legend(fontsize=14)
```

Histogram of $\mathbf{p}^*$, i.e., the true distribution of $p(y=k|x)$. Here we can see not all the input $x$ are so confident. Some of them even has $p(y=k|x)<\frac{1}{3}$, which means the classifier can never achieve 100% accuracy.

```python
_p_corrects = []
for i in range(K_CLAS):
  _p_corrects+=list(np.array(p_true[y_true[:,0]==i,i]))
_p_corrects = np.array(_p_corrects)
plt.hist(_p_corrects,bins=20)
plt.plot([1/3, 1/3],[0,40000],'k-.',label='p=1/3')
plt.xlabel('Probability',fontsize=14)
plt.ylabel('Number of samples',fontsize=14)
#plt.title('Histogram of all the p(y=k|x)',fontsize=14)
plt.legend()
```

# Learn from Gaussian Dataset

Here we train a simple 2-layer MLP with ReLU activation. The hidden size is 128. During training, we will track these metrics:

- **loss**: the training loss of each update

(all of the following metrics are measured at the end of each epoch)

- **tacc**: accuracy on test set
- **vacc**: accuracy on validation set
- **tdistp**: L2-distance between the network's output $\mathbf{q}$ and ground truth distribution $\mathbf{p}^*$ on all training set, i.e., $\|\mathbf{q}(X_{train})-\mathbf{p}^*(X_{train})\|_2$
- **vdistp**: similar to tdistp, but on validation set
- **tdisttgt**: L2-distance between network's output and the supervision it trained on, i.e., $\|\mathbf{q}(X_{train})-\mathbf{p}_{tgt}^*(X_{train})\|_2$
- **vdisttgt**: similar to tdisttgt, but on validation set
- **tECE**: the expected calibration error of network's output on test set, refer to the appendix of the paper for more details
- **vECE**: ECE on validation set
- **L2_p_np**: the L2-distance between the ground truth distribution and its noisy version (later we will let the network directly learn under the supervision of a noisy-p)
- **IDX_MAX**: the number of the best epoch based on validation accuracy

Finally, we also report the **acc, tdistp, ECE** on test set for each model.

```python
def cal_ECE(pb_table, tf_table):
  '''
    pb_table is the probability provided by network
    tf_table is the acc results of the prodiction
  '''
  BM_acc = np.zeros((K_CLAS,))
  BM_conf = np.zeros((K_CLAS,))
  BM_cnt = np.zeros((K_CLAS,))
  Index_table = (pb_table.T*K_CLAS-1e-6).int().squeeze()

  for i in range(pb_table.shape[0]):
    idx = Index_table[i]
    BM_cnt[idx] += 1
    BM_conf[idx] += pb_table[i]
    if tf_table[i]:
      BM_acc[idx] += 1
  ECE = 0
  for j in range(K_CLAS):
    if BM_cnt[j] != 0:
      ECE += BM_cnt[j]*np.abs(BM_acc[j]/BM_cnt[j]-BM_conf[j]/BM_cnt[j])
  return ECE/BM_cnt.sum()

def L2_distance_logits_p(logits, p):
  q = F.softmax(logits,1)
  return L2_distance_q_p(q,p)

def L2_distance_q_p(q, p):
  return  torch.dist(q.reshape(-1,1),p.reshape(-1,1),p=2)#(nn.MSELoss(reduction='mean')(q.reshape(-1,1),p.reshape(-1,1)))**2

def cal_entropy(logits, p):
  # logits is the output of the network
  # p is the true probability (e.g. true probability)
  return nn.KLDivLoss(reduction='batchmean')(F.log_softmax(logits,1), p)

def _y_to_oht(label):
  label_oht = torch.zeros(label.shape[0],K_CLAS).to(label.device)
  label_oht.scatter_(1,label,1)
  label_oht = label_oht.float()
  return label_oht

def _y_to_smoothed(y):
  y_oht = _y_to_oht(y)
  return y_oht*LS_EPS + torch.ones_like(y_oht)*(1-LS_EPS)/K_CLAS

class MLP(nn.Module):
  def __init__(self, in_dim, hid_size=128):
    super(MLP, self).__init__()
    self.in_dim = in_dim
    self.hid_size = hid_size
    self.fc1 = nn.Linear(self.in_dim, self.hid_size)
    self.fc2 = nn.Linear(self.hid_size, self.hid_size)
    self.fc3 = nn.Linear(self.hid_size, K_CLAS)
    self.act = nn.ReLU(True)

  def forward(self, x):
    h1 = self.act(self.fc1(x))
    h2 = self.act(self.fc2(h1))
    out = self.fc3(h2)
    return out
```

```python
def eval_model_on_test(model):
  model.eval()
  for x,y,p in test_loader:
    x,y,p = x.float().cuda(), y.long(), p.float()
    break
  hid = model(x)
  hid = hid.cpu().detach()
  pred = hid.data.max(1, keepdim=True)[1]
  prob_table = torch.gather(nn.Softmax(1)(hid),dim=1,index=pred)
  tf_table = pred.eq(y.data.view_as(pred))
  acc = tf_table.sum()/N_Test
  dist = L2_distance_logits_p(hid, p)
  ECE = cal_ECE(prob_table,tf_table)
  model.train()
  return acc, dist, ECE

def get_validation(model, data_loader, loss_type='from_oht', teacher=None):
  batch_size = data_loader.batch_size
  model.eval()
  correct = 0
  dist_p,dist_tgt = 0, 0
  pb_table, tf_table = [], []
  hid_all, p_all, p_tgt_all = [], [], []
  b_cnt = 0
  for x, y, p in data_loader:
    b_cnt += 1
    x,y,p = x.float().cuda(), y.long(), p.float()
    with torch.no_grad():
      hid = model(x)
      hid = hid.cpu().detach()
      pred = hid.data.max(1, keepdim=True)[1] # get the index of the max log-probability
      prob = torch.gather(nn.Softmax(1)(hid),dim=1,index=pred)
      y_oht=y_to_oht(y.long())
      pb_table.append(prob)
      tf_table.append(pred.eq(y.data.view_as(pred)))

      if loss_type == 'from_oht':
        p_tgt = _y_to_oht(y)
      elif loss_type == 'from_ls':
        p_tgt = _y_to_smoothed(y)
      elif loss_type == 'from_gt' or loss_type == 'noise_prob':
        p_tgt = p
      elif loss_type == 'from_teacher':
        teacher.eval()
        hid_teach = teacher(x)
        hid_teach = hid_teach.cpu().detach()
        p_tgt = F.softmax(hid_teach,1)
      p_all.append(p)
      hid_all.append(hid)
      p_tgt_all.append(p_tgt)
  model.train()
  pb_table = torch.stack(pb_table).reshape(-1,1)
  tf_table = torch.stack(tf_table).reshape(-1,1)
  ECE = cal_ECE(pb_table, tf_table)
  B_NUM = batch_size*b_cnt
  correct = tf_table.sum()

  hid_all = torch.stack(hid_all).reshape(-1,K_CLAS)
  p_all = torch.stack(p_all).reshape(-1,K_CLAS)
  p_tgt_all = torch.stack(p_tgt_all).reshape(-1,K_CLAS)
  dist_p = L2_distance_logits_p(hid_all, p_all)
  dist_tgt = L2_distance_logits_p(hid_all, p_tgt_all)
  return correct/B_NUM, dist_p, dist_tgt, ECE

def train(model, optimizer, scheduler, loss_type='from_oht', teacher=None, prt_flag=False, p_noise=0, noisy_type='combine',temp=1):
  results = {'loss':[], 'tacc':[], 'vacc':[], 'tdistp':[],'vdistp':[],'tdisttgt':[],'vdisttgt':[],'tECE':[],'vECE':[],'L2_ptgt_pgt':[],'IDX_MAX':[]}
  vacc_max = 0
  model.train()
  p_list, p_tgt_list = [], []
  for g in range(EPOCHS):
    for x,y,p in train_loader:
      x,y,p = x.float().cuda(), y.long().cuda(), p.float().cuda()
      optimizer.zero_grad()
      hid = model(x)

      if loss_type == 'from_oht':
        p_tgt = _y_to_oht(y)
      elif loss_type == 'from_ls':
        p_tgt = _y_to_smoothed(y)
      elif loss_type == 'from_gt':
        p_tgt = p
      elif loss_type == 'from_teacher':
        teacher.eval()
        hid_teach = teacher(x)
        hid_teach = hid_teach.detach()
        p_tgt = F.softmax(hid_teach/temp,1)
      elif loss_type == 'noise_prob':
        if noisy_type=='combine':
          p_tgt = _y_to_oht(y)*p_noise[0] + p*(1-p_noise[0])
        elif noisy_type=='plus':
          noisy_p = p + torch.from_numpy(np.random.randn(p.shape[0],p.shape[1])*np.sqrt(p_noise)).cuda()
          noisy_p = torch.clamp(noisy_p,min=1e-6,max=2)
          noisy_p = noisy_p/noisy_p.sum(1).expand([K_CLAS,-1]).transpose(0,1)
          p_tgt = noisy_p.float()
      if g==0:          # Only calculate L2_ptgt_pgt once
        p_list.append(p)
        p_tgt_list.append(p_tgt)
      loss = cal_entropy(hid/temp, p_tgt)/(temp**2)
      loss.backward()
      optimizer.step()
      results['loss'].append(loss.item())
    # ---------- At the end of each epoch ----------
    tacc, tdistp, tdisttgt, tECE = get_validation(model, data_loader=train_loader, loss_type=loss_type, teacher=teacher)
    vacc, vdistp, vdisttgt, vECE = get_validation(model, data_loader=valid_loader, loss_type=loss_type, teacher=teacher)
    results['tacc'].append(tacc)
    results['vacc'].append(vacc)
    results['tdistp'].append(tdistp)
    results['vdistp'].append(vdistp)
    results['tdisttgt'].append(tdisttgt)
    results['vdisttgt'].append(vdisttgt)
    results['tECE'].append(tECE)
    results['vECE'].append(vECE)
    if vacc>vacc_max:
      vacc_max = vacc
      ES_model = copy.deepcopy(model)
      results['IDX_MAX']=g
    if prt_flag and g%10==0:
      print('\t==Training , Epoch: {:3d}/{:3d}\tLoss: {:.6f}\tTACC: {:.6f},\tVACC:{:.6f}'.format(g,EPOCHS, results['loss'][-1], tacc, vacc))
    scheduler.step()
  results['L2_ptgt_pgt'] = L2_distance_q_p(torch.stack(p_list).reshape(-1,K_CLAS), torch.stack(p_tgt_list).reshape(-1,K_CLAS)).cpu()
  return ES_model, results
```

## Experiment 1: verification of Hypothesis 1 (section 2.3 in paper)

Recall hypothesis 1: Suppose we train a model under the supervision of ${p}_\text{tar}$, i.e., minimize $R_\text{tar}(f,\mathcal{D}')$, suppose we have enough training samples, the average L2-distance between ${p}_\text{tar}$ and the ground truth ${p}^*$ on these samples, i.e., $\mathbb{E}_{{x}\sim\mathcal{D}'}\left[\|{p}_\text{tar}({x})-{p}^*({x})\|_2\right]$ is positively correlated with the model's generalization performance.

From hypothesis 1, we might expect the following 2 things happen:

- A neural network trained using better ${p}_\text{tar}$ can have lower accuracy (ACC) and lower expected calibration error (ECE) on test set.
- If the NN is directly trained using ${p}^*$, the performance should be close to a theoretical bound obtained by the optimal Bayesian classifier. (Refer to the appendix A in the paper to find how this bound is calculated.)

In this experiment, we also provide three baselines for comparison:

- **OHT**: learning from one-hot label
- **LS**: learning using label smoothing
- **GT**: learning from ground truth probability
- **KD**: learning from the converged OHT label
- **ESKD**: learning from early-stopped model during the training of OHT model

### Four baselines

```python
OHT_model = MLP(in_dim=X_DIM).cuda()
OHT_optimizer = optim.SGD(OHT_model.parameters(), lr=LR, momentum=0.9)
OHT_scheduler = optim.lr_scheduler.CosineAnnealingLR(OHT_optimizer, T_max=EPOCHS, eta_min=LR_MIN)
best_OHT_model, OHT_results = train(OHT_model, OHT_optimizer, OHT_scheduler, 'from_oht',prt_flag=True)
```

    	==Training , Epoch:   0/100	Loss: 0.783755	TACC: 0.780649,	VACC:0.768800
    	==Training , Epoch:  10/100	Loss: 0.459686	TACC: 0.850962,	VACC:0.840200
    	==Training , Epoch:  20/100	Loss: 0.173808	TACC: 0.859976,	VACC:0.839600
    	==Training , Epoch:  30/100	Loss: 0.438397	TACC: 0.869191,	VACC:0.840200
    	==Training , Epoch:  40/100	Loss: 0.291491	TACC: 0.877604,	VACC:0.838200
    	==Training , Epoch:  50/100	Loss: 0.345821	TACC: 0.884615,	VACC:0.836200
    	==Training , Epoch:  60/100	Loss: 0.171114	TACC: 0.894631,	VACC:0.833200
    	==Training , Epoch:  70/100	Loss: 0.333472	TACC: 0.903646,	VACC:0.832000
    	==Training , Epoch:  80/100	Loss: 0.240411	TACC: 0.911258,	VACC:0.830200
    	==Training , Epoch:  90/100	Loss: 0.429090	TACC: 0.917268,	VACC:0.826000

```python
LS_EPS = (1-0.03)
LS_model = MLP(in_dim=X_DIM).cuda()
LS_optimizer = optim.SGD(LS_model.parameters(), lr=LR, momentum=0.9)
LS_scheduler = optim.lr_scheduler.CosineAnnealingLR(LS_optimizer, T_max=EPOCHS, eta_min=LR_MIN)
best_LS_model, LS_results = train(LS_model, LS_optimizer, LS_scheduler, 'from_ls',prt_flag=True)
```

    	==Training , Epoch:   0/100	Loss: 0.805147	TACC: 0.735577,	VACC:0.720800
    	==Training , Epoch:  10/100	Loss: 0.285832	TACC: 0.847155,	VACC:0.835800
    	==Training , Epoch:  20/100	Loss: 0.361682	TACC: 0.855369,	VACC:0.837400
    	==Training , Epoch:  30/100	Loss: 0.323992	TACC: 0.865385,	VACC:0.837000
    	==Training , Epoch:  40/100	Loss: 0.236004	TACC: 0.874599,	VACC:0.835600
    	==Training , Epoch:  50/100	Loss: 0.242457	TACC: 0.883413,	VACC:0.832600
    	==Training , Epoch:  60/100	Loss: 0.175674	TACC: 0.892829,	VACC:0.833400
    	==Training , Epoch:  70/100	Loss: 0.163700	TACC: 0.901442,	VACC:0.833200
    	==Training , Epoch:  80/100	Loss: 0.199810	TACC: 0.908053,	VACC:0.832200
    	==Training , Epoch:  90/100	Loss: 0.330957	TACC: 0.915465,	VACC:0.828200

```python
GT_model = MLP(in_dim=X_DIM).cuda()
GT_optimizer = optim.SGD(GT_model.parameters(), lr=LR, momentum=0.9)
GT_scheduler = optim.lr_scheduler.CosineAnnealingLR(GT_optimizer, T_max=EPOCHS, eta_min=LR_MIN)
best_GT_model, GT_results = train(GT_model, GT_optimizer, GT_scheduler, 'from_gt',prt_flag=True)
```

    	==Training , Epoch:   0/100	Loss: 0.404297	TACC: 0.741787,	VACC:0.745600
    	==Training , Epoch:  10/100	Loss: 0.017078	TACC: 0.834535,	VACC:0.836000
    	==Training , Epoch:  20/100	Loss: 0.009469	TACC: 0.837740,	VACC:0.838800
    	==Training , Epoch:  30/100	Loss: 0.005915	TACC: 0.838742,	VACC:0.840200
    	==Training , Epoch:  40/100	Loss: 0.013571	TACC: 0.841146,	VACC:0.841800
    	==Training , Epoch:  50/100	Loss: 0.006275	TACC: 0.841546,	VACC:0.842800
    	==Training , Epoch:  60/100	Loss: 0.007221	TACC: 0.842949,	VACC:0.844600
    	==Training , Epoch:  70/100	Loss: 0.007056	TACC: 0.842748,	VACC:0.844600
    	==Training , Epoch:  80/100	Loss: 0.003989	TACC: 0.843750,	VACC:0.844600
    	==Training , Epoch:  90/100	Loss: 0.004542	TACC: 0.842748,	VACC:0.846000

```python
KD_model = MLP(in_dim=X_DIM).cuda()
KD_optimizer = optim.SGD(KD_model.parameters(), lr=LR, momentum=0.9)
KD_scheduler = optim.lr_scheduler.CosineAnnealingLR(KD_optimizer, T_max=EPOCHS, eta_min=LR_MIN)
best_KD_model, KD_results = train(KD_model, KD_optimizer, KD_scheduler, 'from_teacher',teacher=OHT_model,prt_flag=True)
```

    	==Training , Epoch:   0/100	Loss: 0.593286	TACC: 0.731971,	VACC:0.731600
    	==Training , Epoch:  10/100	Loss: 0.164156	TACC: 0.846955,	VACC:0.837800
    	==Training , Epoch:  20/100	Loss: 0.086188	TACC: 0.853165,	VACC:0.839400
    	==Training , Epoch:  30/100	Loss: 0.044294	TACC: 0.858974,	VACC:0.840400
    	==Training , Epoch:  40/100	Loss: 0.062334	TACC: 0.864183,	VACC:0.840200
    	==Training , Epoch:  50/100	Loss: 0.061350	TACC: 0.868590,	VACC:0.838600
    	==Training , Epoch:  60/100	Loss: 0.035931	TACC: 0.871995,	VACC:0.837800
    	==Training , Epoch:  70/100	Loss: 0.049828	TACC: 0.874599,	VACC:0.837600
    	==Training , Epoch:  80/100	Loss: 0.029782	TACC: 0.879006,	VACC:0.836400
    	==Training , Epoch:  90/100	Loss: 0.016668	TACC: 0.882412,	VACC:0.837400

```python
ESKD_model = MLP(in_dim=X_DIM).cuda()
ESKD_optimizer = optim.SGD(ESKD_model.parameters(), lr=LR, momentum=0.9)
ESKD_scheduler = optim.lr_scheduler.CosineAnnealingLR(ESKD_optimizer, T_max=EPOCHS, eta_min=LR_MIN)
best_ESKD_model, ESKD_results = train(ESKD_model, ESKD_optimizer, ESKD_scheduler, 'from_teacher',teacher=best_OHT_model,prt_flag=True)
```

    	==Training , Epoch:   0/100	Loss: 0.432311	TACC: 0.741186,	VACC:0.732800
    	==Training , Epoch:  10/100	Loss: 0.038155	TACC: 0.845753,	VACC:0.840800
    	==Training , Epoch:  20/100	Loss: 0.012359	TACC: 0.845954,	VACC:0.841200
    	==Training , Epoch:  30/100	Loss: 0.037602	TACC: 0.845954,	VACC:0.843200
    	==Training , Epoch:  40/100	Loss: 0.020424	TACC: 0.848157,	VACC:0.843600
    	==Training , Epoch:  50/100	Loss: 0.013502	TACC: 0.847556,	VACC:0.844000
    	==Training , Epoch:  60/100	Loss: 0.015018	TACC: 0.848558,	VACC:0.844400
    	==Training , Epoch:  70/100	Loss: 0.019693	TACC: 0.847957,	VACC:0.845000
    	==Training , Epoch:  80/100	Loss: 0.013214	TACC: 0.847957,	VACC:0.845000
    	==Training , Epoch:  90/100	Loss: 0.011514	TACC: 0.848958,	VACC:0.845400

```python
tmp_oht_q, tmp_ls_q, tmp_kd_q, tmp_eskd_q, tmp_all_p = [], [], [], [], []
for tx,ty,tp in train_loader:
  tx,ty = tx.float().cuda(),ty.long()
  kd_hid = OHT_model(tx).cpu()
  kd_pred = nn.Softmax(1)(kd_hid)
  eskd_hid = best_OHT_model(tx).cpu()
  eskd_pred = nn.Softmax(1)(eskd_hid)
  ls = (_y_to_oht(ty)*LS_EPS+(1-LS_EPS)/(K_CLAS-1)*torch.ones(BATCH_SIZE,K_CLAS))

  tmp_oht_q.append(_y_to_oht(ty))
  tmp_ls_q.append(ls)
  tmp_kd_q.append(kd_pred)
  tmp_eskd_q.append(eskd_pred)
  tmp_all_p.append(tp)
tmp_oht_q = torch.stack(tmp_oht_q).reshape(-1,K_CLAS)
tmp_ls_q = torch.stack(tmp_ls_q).reshape(-1,K_CLAS)
tmp_kd_q = torch.stack(tmp_kd_q).reshape(-1,K_CLAS)
tmp_eskd_q = torch.stack(tmp_eskd_q).reshape(-1,K_CLAS)
tmp_all_p = torch.stack(tmp_all_p).reshape(-1,K_CLAS)

dist_oht_to_p = L2_distance_q_p(tmp_oht_q, tmp_all_p).item()
dist_ls_to_p = L2_distance_q_p(tmp_ls_q, tmp_all_p).item()
dist_kd_to_p = L2_distance_q_p(tmp_kd_q, tmp_all_p).item()
dist_eskd_to_p = L2_distance_q_p(tmp_eskd_q, tmp_all_p).item()
print('OHT\t'+str(dist_oht_to_p))
print('LS\t'+str(dist_ls_to_p))
print('KD\t'+str(dist_kd_to_p))
print('ESKD\t'+str(dist_eskd_to_p))
```

    OHT	33.25286048769626
    LS	32.2937461393822
    KD	16.461855929117295
    ESKD	7.61864426066026

```python
"""
ls_distp, kd_distp, eskd_distp = [],[],[]

TMP_TABLE = [0.5, 1.5, 2, 3, 4]
LS_TABLE = [0.9, 0.85, 0.8, 0.75, 0.7]

def get_ls_dist_q(ls_ratio):
  tmp_ls_q,tmp_all_p=[],[]
  for tx,ty,tp in train_loader:
    ty = ty.long()
    ls = (_y_to_oht(ty)*ls_ratio+(1-ls_ratio)/(K_CLAS-1)*torch.ones(BATCH_SIZE,K_CLAS))
    tmp_ls_q.append(ls)
    tmp_all_p.append(tp)
  tmp_ls_q = torch.stack(tmp_ls_q).reshape(-1,K_CLAS)
  tmp_all_p = torch.stack(tmp_all_p).reshape(-1,K_CLAS)
  return L2_distance_q_p(tmp_ls_q, tmp_all_p).item()

def get_ls_dist_kd(temp):
  tmp_kd_q,tmp_eskd_q,tmp_all_p=[],[],[]
  for tx,ty,tp in train_loader:
    tx,ty = tx.float().cuda(),ty.long()
    kd_hid = OHT_model(tx).cpu()
    kd_pred = nn.Softmax(1)(kd_hid/temp)
    eskd_hid = best_OHT_model(tx).cpu()
    eskd_pred = nn.Softmax(1)(eskd_hid/temp)
    tmp_kd_q.append(kd_pred)
    tmp_eskd_q.append(eskd_pred)
    tmp_all_p.append(tp)
  tmp_kd_q = torch.stack(tmp_kd_q).reshape(-1,K_CLAS)
  tmp_eskd_q = torch.stack(tmp_eskd_q).reshape(-1,K_CLAS)
  tmp_all_p = torch.stack(tmp_all_p).reshape(-1,K_CLAS)
  return L2_distance_q_p(tmp_kd_q, tmp_all_p).item(), L2_distance_q_p(tmp_eskd_q, tmp_all_p).item()

for ls in LS_TABLE:
  ls_distp.append(get_ls_dist_q(ls))

for temp in TMP_TABLE:
  tmp_kd, tmp_eskd = get_ls_dist_kd(temp)
  kd_distp.append(tmp_kd)
  eskd_distp.append(tmp_eskd)
"""
```

```python
def show_test_results(model, type_='OHT'):
  acc, distp, ECE = eval_model_on_test(model)
  print(type_+': \t Test acc is %.5f; tdistp is %.5f; ECE is %.5f'%(acc.item(), distp.item(), ECE.item()))
  return acc, distp, ECE
```

```python
OHT_test_acc, OHT_test_distp, OHT_test_ECE = show_test_results(best_OHT_model, 'OHT ')
LS_test_acc, LS_test_distp, LS_test_ECE = show_test_results(best_LS_model, ' LS ')
GT_test_acc, GT_test_distp, GT_test_ECE = show_test_results(best_GT_model, ' GT ')
KD_test_acc, KD_test_distp, KD_test_ECE = show_test_results(best_KD_model,' KD ')
ESKD_test_acc, ESKD_test_distp, ESKD_test_ECE = show_test_results(best_ESKD_model,'ESKD')
```

    OHT : 	 Test acc is 0.83391; tdistp is 34.04276; ECE is 0.01444
     LS : 	 Test acc is 0.83221; tdistp is 36.87823; ECE is 0.00537
     GT : 	 Test acc is 0.84024; tdistp is 17.24724; ECE is 0.00346
     KD : 	 Test acc is 0.83273; tdistp is 37.12327; ECE is 0.02456
    ESKD: 	 Test acc is 0.83647; tdistp is 27.87039; ECE is 0.00604

```python

```

## Learn from LS/KD/ESKD with different temp.

```python
"""
ls_acc, ls_ECE = [],[]
for i in range(5):
  print(i,end='-')
  LS_EPS = LS_TABLE[i]#(1-0.001*(i+1))
  tmp_model = MLP(in_dim=X_DIM).cuda()
  tmp_optimizer = optim.SGD(tmp_model.parameters(), lr=LR, momentum=0.9)
  tmp_scheduler = optim.lr_scheduler.CosineAnnealingLR(tmp_optimizer, T_max=EPOCHS, eta_min=LR_MIN)
  tmp_best_model, _ = train(tmp_model, tmp_optimizer, tmp_scheduler, 'from_ls',prt_flag=False)
  acc, _, ECE = eval_model_on_test(tmp_best_model)
  ls_acc.append(acc)
  ls_ECE.append(ECE)
  """
```

    0-1-2-3-4-

```python
"""
kd_acc, kd_ECE = [],[]

for i in range(5):
  print(i,end='-')
  temp = TMP_TABLE[i]
  tmp_model = MLP(in_dim=X_DIM).cuda()
  tmp_optimizer = optim.SGD(tmp_model.parameters(), lr=LR, momentum=0.9)
  tmp_scheduler = optim.lr_scheduler.CosineAnnealingLR(tmp_optimizer, T_max=EPOCHS, eta_min=LR_MIN)
  tmp_best_model, _ = train(tmp_model, tmp_optimizer, tmp_scheduler, 'from_teacher',teacher=OHT_model,prt_flag=False,temp=temp)
  acc, _, ECE = eval_model_on_test(tmp_best_model)
  kd_acc.append(acc)
  kd_ECE.append(ECE)
  """
```

    0-

```python
"""
eskd_acc, eskd_ECE = [],[]
for i in range(5):
  print(i,end='-')
  temp = TMP_TABLE[i]
  tmp_model = MLP(in_dim=X_DIM).cuda()
  tmp_optimizer = optim.SGD(tmp_model.parameters(), lr=LR, momentum=0.9)
  tmp_scheduler = optim.lr_scheduler.CosineAnnealingLR(tmp_optimizer, T_max=EPOCHS, eta_min=LR_MIN)
  tmp_best_model, _ = train(tmp_model, tmp_optimizer, tmp_scheduler, 'from_teacher',teacher=best_OHT_model,prt_flag=False,temp=temp)
  acc, _, ECE = eval_model_on_test(tmp_best_model)
  eskd_acc.append(acc)
  eskd_ECE.append(ECE)
  """
```

    0-1-2-3-4-

### Learn from $\mathbf{p}_{tgt}$ with different noise

Type-I noise: combine noise, i.e., $\mathbf{p}_{tgt}=\eta\mathbf{p}^*+(1-\eta)\mathbf{e}_y$ (We use this one in the paper).

Type-II noise: plus noise, i.e., $\mathbf{p}_{tgt}=\text{Norm}[\mathbf{p}^*+\epsilon], \epsilon\sim\mathcal{N}(0,\sigma_p^2)$ as the target probability. $\text{Norm}[x]$ will force $x$ to be a valid categorical distribution. (Older version has result in this one, but we guess the result is hard to explain so we delete it in the camera-ready version.)

```python
test_acc_list_combine = []
test_distp_list_combine = []
test_ECE_list_combine = []
L2_ptgt_pgt_list_combine = []
for i in range(100):
  print(i,end='-')
  noise_level = np.random.uniform(0,1,1)
  S4_model = MLP(in_dim=X_DIM).cuda()
  S4_optimizer = optim.SGD(S4_model.parameters(), lr=LR, momentum=0.9)
  S4_scheduler = optim.lr_scheduler.CosineAnnealingLR(S4_optimizer, T_max=EPOCHS, eta_min=LR_MIN)
  best_S4_model, S4_results = train(S4_model, S4_optimizer, S4_scheduler, 'noise_prob', p_noise=noise_level,noisy_type='combine')
  acc, distp, ECE = eval_model_on_test(best_S4_model)
  test_acc_list_combine.append(acc)
  test_distp_list_combine.append(distp)
  test_ECE_list_combine.append(ECE)
  L2_ptgt_pgt_list_combine.append(S4_results['L2_ptgt_pgt'])

for i in range(20):
  print(i,end='-')
  noise_level = np.random.uniform(0,0.5,1)
  S4_model = MLP(in_dim=X_DIM).cuda()
  S4_optimizer = optim.SGD(S4_model.parameters(), lr=LR, momentum=0.9)
  S4_scheduler = optim.lr_scheduler.CosineAnnealingLR(S4_optimizer, T_max=EPOCHS, eta_min=LR_MIN)
  best_S4_model, S4_results = train(S4_model, S4_optimizer, S4_scheduler, 'noise_prob', p_noise=noise_level,noisy_type='combine')
  acc, distp, ECE = eval_model_on_test(best_S4_model)
  test_acc_list_combine.append(acc)
  test_distp_list_combine.append(distp)
  test_ECE_list_combine.append(ECE)
  L2_ptgt_pgt_list_combine.append(S4_results['L2_ptgt_pgt'])

```

    0-1-2-3-4-5-6-7-8-9-10-11-12-13-14-15-16-17-18-19-20-21-22-23-24-25-26-27-28-29-30-31-32-33-34-35-36-37-38-39-40-41-42-43-44-45-46-47-48-49-50-51-52-53-54-55-56-57-58-59-60-61-62-63-64-65-66-67-68-69-70-71-72-73-74-75-76-77-78-79-80-81-82-83-84-85-86-87-88-89-90-91-92-93-94-95-96-97-98-99-0-1-2-3-4-5-6-7-8-9-10-11-12-13-14-15-16-17-18-19-

```python
np.save('Project1/results/L2_ptgt_pgt_list_combine_N'+str(NOISE)+'.npy',np.array(L2_ptgt_pgt_list_combine))
np.save('Project1/results/test_acc_list_combine_N'+str(NOISE)+'.npy',np.array(test_acc_list_combine))
scipy.stats.spearmanr(L2_ptgt_pgt_list_combine, test_acc_list_combine)
```

    SpearmanrResult(correlation=-0.9309152786843132, pvalue=1.9018749279756831e-53)

```python
#L2_ptgt_pgt_list_combine = np.load('Project1/results/L2_ptgt_pgt_list_combine_N'+str(NOISE)+'.npy')
#test_acc_list_combine = np.load('Project1/results/test_acc_list_combine_N'+str(NOISE)+'.npy')
#test_ECE_list_combine = np.load('Project1/results/test_ECE_list_combine_N'+str(NOISE)+'.npy')
```

```python
plt.scatter(L2_ptgt_pgt_list_combine, test_acc_list_combine, color='blue',alpha=0.6,label='Noisy p')
plt.scatter(dist_oht_to_p,OHT_test_acc,color='green',label='OHT',marker='2',s=400,linewidth=4)
plt.scatter(dist_ls_to_p,LS_test_acc,color='orange',label='LS',marker='2',s=400,linewidth=4)
plt.scatter(0,GT_test_acc,color='red',label='GT',marker='2',s=400,linewidth=4)
plt.scatter(dist_kd_to_p,KD_test_acc,color='cyan',label='KD',marker='2',s=400,linewidth=4)
plt.scatter(dist_eskd_to_p,ESKD_test_acc,color='purple',label='ESKD',marker='2',s=400,linewidth=4)

#plt.scatter(kd_distp,kd_acc,color='cyan',marker='2',s=400,linewidth=4)
#plt.scatter(ls_distp,ls_acc,color='orange',alpha=0.7,marker='2',s=400,linewidth=4)
#plt.scatter(eskd_distp,eskd_acc,color='purple',marker='2',s=400,linewidth=4)

plt.legend(fontsize=12)
plt.ylabel('Accuracy on test set', fontsize=16)
plt.xlabel('L2-distance of p_tar and p*',fontsize=16)
plt.savefig('Project1/Figures/rbt_toy_acc_L2_N'+str(NOISE)+'.pdf')
```

```python
np.save('Project1/results/L2_ptgt_pgt_list_combine_N'+str(NOISE)+'.npy',np.array(L2_ptgt_pgt_list_combine))
np.save('Project1/results/test_ECE_list_combine_N'+str(NOISE)+'.npy',np.array(test_ECE_list_combine))
scipy.stats.spearmanr(L2_ptgt_pgt_list_combine, test_ECE_list_combine)
```

    SpearmanrResult(correlation=0.8954719077713729, pvalue=2.705262593852301e-43)

```python
plt.scatter(L2_ptgt_pgt_list_combine,test_ECE_list_combine, color='blue',alpha=0.6,label='Noisy p')
plt.scatter(OHT_results['L2_ptgt_pgt'],OHT_results['vECE'][OHT_results['IDX_MAX']],color='green',label='OHT',marker='2',s=400,linewidth=4)
plt.scatter(LS_results['L2_ptgt_pgt'],LS_results['vECE'][LS_results['IDX_MAX']], color='orange',label='LS',marker='2',s=400,linewidth=4)
plt.scatter(GT_results['L2_ptgt_pgt'],GT_results['vECE'][GT_results['IDX_MAX']], color='red',label='GT',marker='2',s=400,linewidth=4)
plt.scatter(KD_results['L2_ptgt_pgt'],KD_results['vECE'][KD_results['IDX_MAX']], color='cyan',label='KD',marker='2',s=400,linewidth=4)
plt.scatter(ESKD_results['L2_ptgt_pgt'],ESKD_results['vECE'][ESKD_results['IDX_MAX']], color='purple',label='ESKD',marker='2',s=400,linewidth=4)

#plt.scatter(kd_distp,kd_ECE,color='cyan',marker='2',s=400,linewidth=4)
#plt.scatter(ls_distp,ls_ECE,color='orange',alpha=0.7,marker='2',s=400,linewidth=4)
#plt.scatter(eskd_distp,eskd_ECE,color='purple',marker='2',s=400,linewidth=4)
plt.legend(fontsize=12)
#plt.xlim(-0.02,0.3)
#plt.ylim(-0.02,0.21)

plt.ylabel('ECE on test set', fontsize=16)
plt.xlabel('L2-distance of p_tar and p*',fontsize=16)
plt.savefig('Project1/Figures/rbt_toy_ECE_L2_N'+str(NOISE)+'.pdf')
```


1. [Create all value pairs for v1 and v2](#AllValuePairs)
* [Replace NaN with mode](#NaN2Mode)
* [Use sample builtin function to create sample from matrix](#sample)
* [Count of Matching Values in two Matrices/Vectors](#MatchinRows)
* [Cross Validation](#CrossValidation)
* [Value-based join of two Matrices](#JoinMatrices)
* [Filter Matrix to include only Frequent Column Values](#FilterMatrix)
* [(Sparse) Matrix to/from (rowIndex, colIndex, values) conversions (i,j,v)](#Construct_sparse_Matrix)
* [Find and remove duplicates in columns or rows](#Find_and_remove_duplicates)
* [Set based Indexing](#Set_based_Indexing)
* [Group by Aggregate using Linear Algebra](#Multi_column_Sorting)
* [Cumulative Summation with Decay Multiplier](#CumSum_Product)
* [Invert Lower Triangular Matrix](#Invert_Lower_Triangular_Matrix)

```python
from systemml import MLContext, dml
ml = MLContext(sc)

print (ml.buildTime())
```

## Create all value pairs for v1 and v2

```python
prog="""
v1 = matrix ('2 1 8 3 5 6 7', rows = 7, cols = 1 )
v2 = matrix ('80 20 50', rows = 3, cols = 1 )

nv1 = nrow (v1);
nv2 = nrow (v2);
R = cbind (
  matrix (v1 %*% matrix(1, 1, nv2), nv1*nv2, 1),
  matrix (matrix(1, nv1, 1) %*% t(v2), nv1*nv2, 1))

print(toString(v1));
print(toString(v2));
print(toString(R));
"""
res = ml.execute(dml(prog))
```

    2.000
    1.000
    8.000
    3.000
    5.000
    6.000
    7.000

    80.000
    20.000
    50.000

    2.000 80.000
    2.000 20.000
    2.000 50.000
    1.000 80.000
    1.000 20.000
    1.000 50.000
    8.000 80.000
    8.000 20.000
    8.000 50.000
    3.000 80.000
    3.000 20.000
    3.000 50.000
    5.000 80.000
    5.000 20.000
    5.000 50.000
    6.000 80.000
    6.000 20.000
    6.000 50.000
    7.000 80.000
    7.000 20.000
    7.000 50.000

    SystemML Statistics:
    Total execution time:		0.000 sec.
    Number of executed Spark inst:	0.

## Replace NaN with mode

This functions replaces NaN in column i with mode of column i.

```python
prog="""
# Function for NaN-aware replacement with mode
replaceNaNwithMode = function (matrix[double] X, integer colId)
      return (matrix[double] X)
{
   Xi = replace (target=X[,colId], pattern=NaN, replacement=-Inf)               # replace NaN with -Inf
   Xi = replace (target=Xi, pattern=-Inf, replacement=max(Xi)+1)                # replace -Inf with largest value + 1
   agg = aggregate (target=Xi, groups=Xi, fn="count")                           # count each distinct value
   mode = as.scalar (rowIndexMax(t(agg[1:nrow(agg)-1, ])))                      # mode is max frequent value except last value
   X[,colId] = replace (target=Xi, pattern=max(Xi), replacement=mode)           # fill in mode
}

X = matrix('1 NaN 1 NaN 1 2 2 1 1 2', rows = 5, cols = 2)

Y = replaceNaNwithMode (X, 2)

print ("Before: \n" + toString(X))
print ("After: \n" + toString(Y))
"""
res = ml.execute(dml(prog))
```

    Before:
    1.000 NaN
    1.000 NaN
    1.000 2.000
    2.000 1.000
    1.000 2.000

    After:
    1.000 2.000
    1.000 2.000
    1.000 2.000
    2.000 1.000
    1.000 2.000

    SystemML Statistics:
    Total execution time:		0.001 sec.
    Number of executed Spark inst:	0.

## Use sample builtin function to create sample from matrix

Use sample() function, create permutation matrix using table(), and pull sample from X.

```python
prog="""
X = matrix ('2 1 8 3 5 6 7 9 4 4', rows = 5, cols = 2 )

nbrSamples = 2

sv = order (target = sample (nrow (X), nbrSamples, FALSE))  # samples w/o replacement, and order
P = table (seq (1, nbrSamples), sv, nbrSamples, nrow(X))    # permutation matrix
samples = P %*% X;                                          # apply P to perform selection

print ("X: \n" + toString(X))
print ("sv: \n" + toString(sv))
print ("samples: \n" + toString(samples))
"""
res = ml.execute(dml(prog))
```

    X:
    2.000 1.000
    8.000 3.000
    5.000 6.000
    7.000 9.000
    4.000 4.000

    sv:
    1.000
    5.000

    samples:
    2.000 1.000
    4.000 4.000

    SystemML Statistics:
    Total execution time:		0.000 sec.
    Number of executed Spark inst:	0.

## Count of Matching Values in two Matrices/Vectors

Given two matrices/vectors X and Y, get a count of the rows where X and Y have the same value.

```python
prog="""
X = matrix('8 4 5 4 9 10', rows = 6, cols = 1)
Y = matrix('4 9 5 1 9 7 ', rows = 6, cols = 1)

matches = sum (X == Y)

print ("t(X): " + toString(t(X)))
print ("t(Y): " + toString(t(Y)))
print ("Number of Matches: " + matches + "\n")
"""
res = ml.execute(dml(prog))
```

    t(X): 8.000 4.000 5.000 4.000 9.000 10.000

    t(Y): 4.000 9.000 5.000 1.000 9.000 7.000

    Number of Matches: 2.0

    SystemML Statistics:
    Total execution time:		0.000 sec.
    Number of executed Spark inst:	0.

## Cross Validation

Perform kFold cross validation by running in parallel fold creation, training algorithm, test algorithm, and evaluation.

```python
prog = """
holdOut = 1/3
kFolds = 1/holdOut

nRows = 6; nCols = 3;

X = matrix(seq(1, nRows * nCols), rows = nRows, cols = nCols)             # X data
y = matrix(seq(1, nRows), rows = nRows, cols = 1)                         # y label data
Xy = cbind (X,y)                                                          # Xy Data for CV

sv = rand (rows = nRows, cols = 1, min = 0.0, max = 1.0, pdf = "uniform") # sv selection vector for fold creation
sv = (order(target=sv, by=1, index.return=TRUE)) %% kFolds + 1            #    with numbers between 1 .. kFolds

stats = matrix(0, rows=kFolds, cols=1)                                    # stats per kFolds model on test data

parfor (i in 1:kFolds)
{
   # Skip empty training data or test data.
   if  ( sum (sv == i) > 0 & sum (sv == i) < nrow(X) )
   {
      Xyi  = removeEmpty(target = Xy, margin = "rows", select = (sv == i))  # Xyi fold, i.e. 1/k of rows (test data)
      Xyni = removeEmpty(target = Xy, margin = "rows", select = (sv != i))  # Xyni data, i.e. (k-1)/k of rows (train data)

      # Skip extreme label inbalance
      distinctLabels = aggregate( target = Xyni[,1], groups = Xyni[,1], fn = "count")
      if ( nrow(distinctLabels) > 1)
      {
         w_i = trainAlg (Xyni[ ,1:ncol(Xy)-1], Xyni[ ,ncol(Xy)])             # w_i Model for i-th training data
         p_i = testAlg  (Xyi [ ,1:ncol(Xy)-1], w_i)                          # p_i Prediction for i-th test data
         e_i = evalPrediction (p_i, Xyi[ ,ncol(Xy)])                         # stats[i,] evaluation of prediction of i-th fold
         stats[i,] = e_i

         print (  "Test data Xyi" + i + "\n" + toString(Xyi)
               + "\nTrain data Xyni" + i + "\n" + toString(Xyni)
               + "\nw_" + i + "\n" + toString(w_i)
               + "\nstats" + i + "\n" + toString(stats[i,])
               + "\n")
      }
      else
      {
        print ("Training data for fold " + i + " has only " + nrow(distinctLabels) + " distinct labels. Needs to be > 1.")
      }
   }
   else
   {
      print ("Training data or test data for fold " + i + " is empty. Fold not validated.")
   }

}

print ("SV selection vector:\n" + toString(sv))

trainAlg = function (matrix[double] X, matrix[double] y)
  return (matrix[double] w)
{
   w = t(X) %*% y
}

testAlg = function (matrix[double] X, matrix[double] w)
  return (matrix[double] p)
{
   p = X %*% w
}

evalPrediction = function (matrix[double] p, matrix[double] y)
  return (matrix[double] e)
{
   e = as.matrix(sum (p - y))
}
"""

res = ml.execute(dml(prog))
```

    Test data Xyi1
    7.000 8.000 9.000 3.000
    10.000 11.000 12.000 4.000

    Train data Xyni1
    1.000 2.000 3.000 1.000
    4.000 5.000 6.000 2.000
    13.000 14.000 15.000 5.000
    16.000 17.000 18.000 6.000

    w_1
    170.000
    184.000
    198.000

    stats1
    10537.000

    Test data Xyi2
    13.000 14.000 15.000 5.000
    16.000 17.000 18.000 6.000

    Train data Xyni2
    1.000 2.000 3.000 1.000
    4.000 5.000 6.000 2.000
    7.000 8.000 9.000 3.000
    10.000 11.000 12.000 4.000

    w_2
    70.000
    80.000
    90.000

    stats2
    7469.000

    Test data Xyi3
    1.000 2.000 3.000 1.000
    4.000 5.000 6.000 2.000

    Train data Xyni3
    7.000 8.000 9.000 3.000
    10.000 11.000 12.000 4.000
    13.000 14.000 15.000 5.000
    16.000 17.000 18.000 6.000

    w_3
    222.000
    240.000
    258.000

    stats3
    5109.000

    SV selection vector:
    3.000
    3.000
    1.000
    1.000
    2.000
    2.000

    SystemML Statistics:
    Total execution time:		0.014 sec.
    Number of executed Spark inst:	0.

## Value-based join of two Matrices

Given matrix M1 and M2, join M1 on column 2 with M2 on column 2, and return matching rows of M1.

```python
prog = """
M1 = matrix ('1 1 2 3 3 3 4 4 5 3 6 4 7 1 8 2 9 1', rows = 9, cols = 2)
M2 = matrix ('1 1 2 8 3 3 4 3 5 1', rows = 5, cols = 2)

I = rowSums (outer (M1[,2], t(M2[,2]), "=="))                  # I : indicator matrix for M1
M12 = removeEmpty (target = M1, margin = "rows", select = I)   # apply filter to retrieve join result

print ("M1 \n" + toString(M1))
print ("M2 \n" + toString(M2))
print ("M1[,2] joined with M2[,2], and return matching M1 rows\n" + toString(M12))
"""
res = ml.execute(dml(prog))
```

    M1
    1.000 1.000
    2.000 3.000
    3.000 3.000
    4.000 4.000
    5.000 3.000
    6.000 4.000
    7.000 1.000
    8.000 2.000
    9.000 1.000

    M2
    1.000 1.000
    2.000 8.000
    3.000 3.000
    4.000 3.000
    5.000 1.000

    M1[,2] joined with M2[,2], and return matching M1 rows
    1.000 1.000
    2.000 3.000
    3.000 3.000
    5.000 3.000
    7.000 1.000
    9.000 1.000

    SystemML Statistics:
    Total execution time:		0.001 sec.
    Number of executed Spark inst:	0.

## Filter Matrix to include only Frequent Column Values

Given a matrix, filter the matrix to only include rows with column values that appear more often than MinFreq.

```python
prog = """
MinFreq = 3                                                           # minimum frequency of tokens

M = matrix ('1 1 2 3 3 3 4 4 5 3 6 4 7 1 8 2 9 1', rows = 9, cols = 2)

gM = aggregate (target = M[,2], groups = M[,2], fn = "count")         # gM: group by and count (grouped matrix)
gv = cbind (seq(1,nrow(gM)), gM)                                      # gv: add group values to counts (group values)
fg = removeEmpty (target = gv * (gv[,2] >= MinFreq), margin = "rows") # fg: filtered groups
I = rowSums (outer (M[,2] ,t(fg[,1]), "=="))                          # I : indicator of size M with filtered groups
fM = removeEmpty (target = M, margin = "rows", select = I)            # FM: filter matrix

print (toString(M))
print (toString(fM))
"""
res = ml.execute(dml(prog))
```

    1.000 1.000
    2.000 3.000
    3.000 3.000
    4.000 4.000
    5.000 3.000
    6.000 4.000
    7.000 1.000
    8.000 2.000
    9.000 1.000

    1.000 1.000
    2.000 3.000
    3.000 3.000
    5.000 3.000
    7.000 1.000
    9.000 1.000

    SystemML Statistics:
    Total execution time:		0.001 sec.
    Number of executed Spark inst:	0.

## (Sparse) Matrix to/from (rowIndex, colIndex, values) conversions (i,j,v)

Given rowIndex, colIndex, and values as column vectors, construct (sparse) matrix.

```python
prog = """
I = matrix ("1 3 3 4 5", rows = 5, cols = 1)
J = matrix ("2 3 4 1 6", rows = 5, cols = 1)
V = matrix ("10 20 30 40 50", rows = 5, cols = 1)

IJVs = cbind(I, J, V)

M = table (I, J, V)

print (toString (IJVs))
print (toString (M))
"""
res = ml.execute(dml(prog).output('M')).get('M').toNumPy()
```

    1.000 2.000 10.000
    3.000 3.000 20.000
    3.000 4.000 30.000
    4.000 1.000 40.000
    5.000 6.000 50.000

    0.000 10.000 0.000 0.000 0.000 0.000
    0.000 0.000 0.000 0.000 0.000 0.000
    0.000 0.000 20.000 30.000 0.000 0.000
    40.000 0.000 0.000 0.000 0.000 0.000
    0.000 0.000 0.000 0.000 0.000 50.000

    SystemML Statistics:
    Total execution time:		0.001 sec.
    Number of executed Spark inst:	0.

Given a sparse matrix, construct ```` matrix with 3 columns rowIndex, colIndex, and values.

```python
prog = """
M = matrix ("0 23  10 0 18 0 0 20", rows = 4, cols = 2)

m = nrow(M);
n = ncol(M);
I = matrix((M!=0)*seq(1,m), m*n, 1)
J = matrix((M!=0)*t(seq(1,n)), m*n, 1)
V = matrix(M, m*n, 1)
IJVd = cbind(I, J, V);
IJVs = removeEmpty(target=IJVd, margin="rows");

print ("M:\n" + toString(M))
print ("IJVs:\n" + toString (IJVs))
"""
res = ml.execute(dml(prog).output('M')).get('M').toNumPy()
```

    M:
    0.000 23.000
    10.000 0.000
    18.000 0.000
    0.000 20.000

    IJVs:
    1.000 2.000 23.000
    2.000 1.000 10.000
    3.000 1.000 18.000
    4.000 2.000 20.000

    SystemML Statistics:
    Total execution time:		0.001 sec.
    Number of executed Spark inst:	0.

## Find and remove duplicates in columns or rows

### Assuming values are sorted.

```python
prog = """
X = matrix ("1 2 3 3 3 4 5 10", rows = 8, cols = 1)

I = rbind (matrix (1,1,1), (X[1:nrow (X)-1,] != X[2:nrow (X),]));             # compare current with next value
res = removeEmpty (target = X, margin = "rows", select = I);                  # select where different
"""
ml.execute(dml(prog).output('res')).get('res').toNumPy()
```

    SystemML Statistics:
    Total execution time:		0.000 sec.
    Number of executed Spark inst:	0.

    array([[  1.],
           [  2.],
           [  3.],
           [  4.],
           [  5.],
           [ 10.]])

### No assumptions on values.

```python
prog = """
X = matrix ("3 2 1 3 3 4 5 10", rows = 8, cols = 1)

I = aggregate (target = X, groups = X[,1], fn = "count")                                 # group and count duplicates
res = removeEmpty (target = seq (1, max (X[,1])), margin = "rows", select = (I != 0));   # select groups
"""
ml.execute(dml(prog).output('res')).get('res').toNumPy()
```

    SystemML Statistics:
    Total execution time:		0.076 sec.
    Number of executed Spark inst:	6.

    array([[  1.],
           [  2.],
           [  3.],
           [  4.],
           [  5.],
           [ 10.]])

### Order the values and then remove duplicates.

```python
prog = """
X = matrix ("3 2 1 3 3 4 5 10", rows = 8, cols = 1)

X = order (target = X, by = 1)                                      # order values
I = rbind (matrix (1,1,1), (X[1:nrow (X)-1,] != X[2:nrow (X),]));
res = removeEmpty (target = X, margin = "rows", select = I);
"""
ml.execute(dml(prog).output('res')).get('res').toNumPy()
```

    SystemML Statistics:
    Total execution time:		0.000 sec.
    Number of executed Spark inst:	0.

    array([[  1.],
           [  2.],
           [  3.],
           [  4.],
           [  5.],
           [ 10.]])

## Set based Indexing

Given a matrix X, and a indicator matrix J with indices into X.
Use J to perform operation on X, e.g. add value 10 to cells in X indicated by J.

```python
prog = """
X = matrix (1, rows = 1, cols = 100)
J = matrix ("10 20 25 26 28 31 50 67 79", rows = 1, cols = 9)

res = X + table (matrix (1, rows = 1, cols = ncol (J)), J, 10)

print (toString (res))
"""
ml.execute(dml(prog).output('res')).get('res').toNumPy()
```

    1.000 1.000 1.000 1.000 1.000 1.000 1.000 1.000 1.000 11.000 1.000 1.000 1.000 1.000 1.000 1.000 1.000 1.000 1.000 11.000 1.000 1.000 1.000 1.000 11.000 11.000 1.000 11.000 1.000 1.000 11.000 1.000 1.000 1.000 1.000 1.000 1.000 1.000 1.000 1.000 1.000 1.000 1.000 1.000 1.000 1.000 1.000 1.000 1.000 11.000 1.000 1.000 1.000 1.000 1.000 1.000 1.000 1.000 1.000 1.000 1.000 1.000 1.000 1.000 1.000 1.000 11.000 1.000 1.000 1.000 1.000 1.000 1.000 1.000 1.000 1.000 1.000 1.000 11.000

    SystemML Statistics:
    Total execution time:		0.001 sec.
    Number of executed Spark inst:	0.

    array([[  1.,   1.,   1.,   1.,   1.,   1.,   1.,   1.,   1.,  11.,   1.,
              1.,   1.,   1.,   1.,   1.,   1.,   1.,   1.,  11.,   1.,   1.,
              1.,   1.,  11.,  11.,   1.,  11.,   1.,   1.,  11.,   1.,   1.,
              1.,   1.,   1.,   1.,   1.,   1.,   1.,   1.,   1.,   1.,   1.,
              1.,   1.,   1.,   1.,   1.,  11.,   1.,   1.,   1.,   1.,   1.,
              1.,   1.,   1.,   1.,   1.,   1.,   1.,   1.,   1.,   1.,   1.,
             11.,   1.,   1.,   1.,   1.,   1.,   1.,   1.,   1.,   1.,   1.,
              1.,  11.]])

## Group by Aggregate using Linear Algebra

Given a matrix PCV as (Position, Category, Value), sort PCV by category, and within each category by value in descending order. Create indicator vector for category changes, create distinct categories, and perform linear algebra operations.

```python
prog = """
C = matrix ('50 40 20 10 30 20 40 20 30', rows = 9, cols = 1)                              # category data
V = matrix ('20 11 49 33 94 29 48 74 57', rows = 9, cols = 1)                              # value data

PCV = cbind (cbind (seq (1, nrow (C), 1), C), V);                                      # PCV representation
PCV = order (target = PCV, by = 3, decreasing = TRUE,  index.return = FALSE);
PCV = order (target = PCV, by = 2, decreasing = FALSE, index.return = FALSE);

# Find all rows of PCV where the category has a new value, in comparison to the previous row

is_new_C = matrix (1, rows = 1, cols = 1);
if (nrow (C) > 1) {
  is_new_C = rbind (is_new_C, (PCV [1:nrow(C) - 1, 2] < PCV [2:nrow(C), 2]));
}

# Associate each category with its index

index_C = cumsum (is_new_C);                                                          # cumsum

# For each category, compute:
#   - the list of distinct categories
#   - the maximum value for each category
#   - 0-1 aggregation matrix that adds records of the same category

distinct_C  = removeEmpty (target = PCV [, 2], margin = "rows", select = is_new_C);
max_V_per_C = removeEmpty (target = PCV [, 3], margin = "rows", select = is_new_C);
C_indicator = table (index_C, PCV [, 1], max (index_C), nrow (C));                    # table

sum_V_per_C = C_indicator %*% V
"""

res = ml.execute(dml(prog).output('PCV','distinct_C', 'max_V_per_C', 'C_indicator', 'sum_V_per_C'))
print (res.get('PCV').toNumPy())
print (res.get('distinct_C').toNumPy())
print (res.get('max_V_per_C').toNumPy())
print (res.get('C_indicator').toNumPy())
print (res.get('sum_V_per_C').toNumPy())
```

    SystemML Statistics:
    Total execution time:		0.002 sec.
    Number of executed Spark inst:	0.

    [[  4.  10.  33.]
     [  8.  20.  74.]
     [  3.  20.  49.]
     [  6.  20.  29.]
     [  5.  30.  94.]
     [  9.  30.  57.]
     [  7.  40.  48.]
     [  2.  40.  11.]
     [  1.  50.  20.]]
    [[ 10.]
     [ 20.]
     [ 30.]
     [ 40.]
     [ 50.]]
    [[ 33.]
     [ 74.]
     [ 94.]
     [ 48.]
     [ 20.]]
    [[ 0.  0.  0.  1.  0.  0.  0.  0.  0.]
     [ 0.  0.  1.  0.  0.  1.  0.  1.  0.]
     [ 0.  0.  0.  0.  1.  0.  0.  0.  1.]
     [ 0.  1.  0.  0.  0.  0.  1.  0.  0.]
     [ 1.  0.  0.  0.  0.  0.  0.  0.  0.]]
    [[  33.]
     [ 152.]
     [ 151.]
     [  59.]
     [  20.]]

## Cumulative Summation with Decay Multiplier

Given matrix X, compute:

    Y[i] = X[i]
         + X[i-1] * C[i]
         + X[i-2] * C[i] * C[i-1]
         + X[i-3] * C[i] * C[i-1] * C[i-2]
         + ...

```python
cumsum_prod_def = """
cumsum_prod = function (Matrix[double] X, Matrix[double] C, double start)
  return (Matrix[double] Y)
# Computes the following recurrence in log-number of steps:
#   Y [1, ] = X [1, ] + C [1, ] * start;
#   Y [i+1, ] = X [i+1, ] + C [i+1, ] * Y [i, ]
{
   Y = X;  P = C;  m = nrow(X);  k = 1;
   Y [1,] = Y [1,] + C [1,] * start;
   while (k < m) {
     Y [k + 1:m,] = Y [k + 1:m,] + Y [1:m - k,] * P [k + 1:m,];
     P [k + 1:m,] = P [1:m - k,] * P [k + 1:m,];
     k = 2 * k;
   }
}
"""
```

In this example we use cumsum_prod for cumulative summation with "breaks", that is, multiple cumulative summations in one.

```python
prog = cumsum_prod_def + """
X = matrix ("1 2 3 4 5 6 7 8 9", rows = 9, cols = 1);

#Zeros in C cause "breaks" that restart the cumulative summation from 0
C = matrix ("0 1 1 0 1 1 1 0 1", rows = 9, cols = 1);

Y = cumsum_prod (X, C, 0);

print (toString(Y))
"""
ml.execute(dml(prog))
```

    1.000
    3.000
    6.000
    4.000
    9.000
    15.000
    22.000
    8.000
    17.000

    SystemML Statistics:
    Total execution time:		0.001 sec.
    Number of executed Spark inst:	0.

    MLResults

In this example, we copy selected rows downward to all consecutive non-selected rows.

```python
prog = cumsum_prod_def + """
X = matrix ("1 2 3 4 5 6 7 8 9", rows = 9, cols = 1);

# Ones in S represent selected rows to be copied, zeros represent non-selected rows
S = matrix ("1 0 0 1 0 0 0 1 0", rows = 9, cols = 1);

Y = cumsum_prod (X * S, 1 - S, 0);

print (toString(Y))
"""
ml.execute(dml(prog))
```

    1.000
    1.000
    1.000
    4.000
    4.000
    4.000
    4.000
    8.000
    8.000

    SystemML Statistics:
    Total execution time:		0.001 sec.
    Number of executed Spark inst:	0.

    MLResults

This is a naive implementation of cumulative summation with decay multiplier.

```python
cumsum_prod_naive_def = """
cumsum_prod_naive = function (Matrix[double] X, Matrix[double] C, double start)
  return (Matrix[double] Y)
{
  Y = matrix (0, rows = nrow(X), cols = ncol(X));
  Y [1,] = X [1,] + C [1,] * start;
  for (i in 2:nrow(X))
  {
    Y [i,] = X [i,] + C [i,] * Y [i - 1,]
  }
}
"""
```

There is a significant performance difference between the naive implementation and the tricky implementation.

```python
prog = cumsum_prod_def + cumsum_prod_naive_def + """
X = rand (rows = 20000, cols = 10, min = 0, max = 1, pdf = "uniform", sparsity = 1.0);
C = rand (rows = 20000, cols = 10, min = 0, max = 1, pdf = "uniform", sparsity = 1.0);

Y1 = cumsum_prod_naive (X, C, 0.123);
"""
ml.execute(dml(prog))
```

    SystemML Statistics:
    Total execution time:		6.081 sec.
    Number of executed Spark inst:	0.

    MLResults

```python
prog = cumsum_prod_def + cumsum_prod_naive_def + """
X = rand (rows = 20000, cols = 10, min = 0, max = 1, pdf = "uniform", sparsity = 1.0);
C = rand (rows = 20000, cols = 10, min = 0, max = 1, pdf = "uniform", sparsity = 1.0);

Y2 = cumsum_prod (X, C, 0.123);
"""
ml.execute(dml(prog))
```

    SystemML Statistics:
    Total execution time:		0.074 sec.
    Number of executed Spark inst:	0.

    MLResults

## Invert Lower Triangular Matrix

In this example, we invert a lower triangular matrix using a the following divide-and-conquer approach. Given lower triangular matrix L, we compute its inverse X which is also lower triangular by splitting both matrices in the middle into 4 blocks (in a 2x2 fashion), and multiplying them together to get the identity matrix:

\begin{equation}
L \text{ %*% } X = \left(\begin{matrix} L_1 & 0 \\ L_2 & L_3 \end{matrix}\right)
\text{ %*% } \left(\begin{matrix} X_1 & 0 \\ X_2 & X_3 \end{matrix}\right)
= \left(\begin{matrix} L_1 X_1 & 0 \\ L_2 X_1 + L_3 X_2 & L_3 X_3 \end{matrix}\right)
= \left(\begin{matrix} I & 0 \\ 0 & I \end{matrix}\right)
\nonumber
\end{equation}

If we multiply blockwise, we get three equations:

$
\begin{equation}
L1 \text{ %*% } X1 = 1\\
L3 \text{ %*% } X3 = 1\\
L2 \text{ %*% } X1 + L3 \text{ %*% } X2 = 0\\
\end{equation}
$

Solving these equation gives the following formulas for X:

$
\begin{equation}
X1 = inv(L1) \\
X3 = inv(L3) \\
X2 = - X3 \text{ %*% } L2 \text{ %*% } X1 \\
\end{equation}
$

If we already recursively inverted L1 and L3, we can invert L2.  This suggests an algorithm that starts at the diagonal and iterates away from the diagonal, involving bigger and bigger blocks (of size 1, 2, 4, 8, etc.)  There is a logarithmic number of steps, and inside each step, the inversions can be performed in parallel using a parfor-loop.

Function "invert_lower_triangular" occurs within more general inverse operations and matrix decompositions.  The divide-and-conquer idea allows to derive more efficient algorithms for other matrix decompositions.

```python
invert_lower_triangular_def = """
invert_lower_triangular = function (Matrix[double] LI)
  return   (Matrix[double] LO)
{
  n = nrow (LI);
  LO = matrix (0, rows = n, cols = n);
  LO = LO + diag (1 / diag (LI));

  k = 1;
  while (k < n)
  {
    LPF = matrix (0, rows = n, cols = n);
    parfor (p in 0:((n - 1) / (2 * k)), check = 0)
    {
    i = 2 * k * p;
    j = i + k;
    q = min (n, j + k);
    if (j + 1 <= q) {
      L1 = LO [i + 1:j, i + 1:j];
      L2 = LI [j + 1:q, i + 1:j];
      L3 = LO [j + 1:q, j + 1:q];
      LPF [j + 1:q, i + 1:j] = -L3 %*% L2 %*% L1;
    }
    }
    LO = LO + LPF;
    k = 2 * k;
  }
}
"""
```

```python
prog =  invert_lower_triangular_def + """
n = 1000;
A = rand (rows = n, cols = n, min = -1, max = 1, pdf = "uniform", sparsity = 1.0);
Mask = cumsum (diag (matrix (1, rows = n, cols = 1)));

L = (A %*% t(A)) * Mask;     # Generate L for stability of the inverse

X = invert_lower_triangular (L);

print ("Maximum difference between X %*% L and Identity = " + max (abs (X %*% L - diag (matrix (1, rows = n, cols = 1)))));
"""
ml.execute(dml(prog))
```

    Maximum difference between X %*% L and Identity = 2.220446049250313E-16
    SystemML Statistics:
    Total execution time:		0.309 sec.
    Number of executed Spark inst:	0.

    MLResults

This is a naive implementation of inverting a lower triangular matrix.

```python
invert_lower_triangular_naive_def = """
invert_lower_triangular_naive = function (Matrix[double] LI)
  return   (Matrix[double] LO)
{
  n = nrow (LI);
  LO = diag (matrix (1, rows = n, cols = 1));
  for (i in 1:n - 1)
  {
    LO [i,] = LO [i,] / LI [i, i];
    LO [i + 1:n,] = LO [i + 1:n,] - LI [i + 1:n, i] %*% LO [i,];
  }
  LO [n,] = LO [n,] / LI [n, n];
}
"""
```

The naive implementation is significantly slower than the divide-and-conquer implementation.

```python
prog =  invert_lower_triangular_naive_def + """
n = 1000;
A = rand (rows = n, cols = n, min = -1, max = 1, pdf = "uniform", sparsity = 1.0);
Mask = cumsum (diag (matrix (1, rows = n, cols = 1)));

L = (A %*% t(A)) * Mask;     # Generate L for stability of the inverse

X = invert_lower_triangular_naive (L);

print ("Maximum difference between X %*% L and Identity = " + max (abs (X %*% L - diag (matrix (1, rows = n, cols = 1)))));
"""
ml.execute(dml(prog))
```

    Maximum difference between X %*% L and Identity = 4.718447854656915E-16
    SystemML Statistics:
    Total execution time:		6.890 sec.
    Number of executed Spark inst:	0.

    MLResults

```python

```

samples/jupyter-notebooks/DML Tips and Tricks (aka Fun With DML).ipynb


    Paradigma de programação: Uma forma de abordar um problema do ponto de vista
    lógico e prático no momento de codar, padrão de comportamento

    Esse paradigma trata a programação de um ponto de vista
    matemático...

    Mas o que isso significa?

Ponto de vita imperativo

```python
a = int(input('Primeiro numero: '))
b = int(input('Segundo numero: '))
operacao = input('Qual a operação')

if operacao == '+':
  print(a+b)
else:
  print("Operação invalida no momento")
```

Ponto de vista do objeto

    1° Construir um objeto
    2° Gerar os atributos desse objeto
    3° Gerar as ações desse objeto

    De maneira geral, a chamada da POO também é difere da
    programação funcional

```python
class Calculadora():
  def __init__(self, primeiroNumero, segundoNumero):
    self.primeiroNumero = primeiroNumero
    self.segundoNumero = segundoNumero

  def soma(self):
    return self.primeiroNumero + self.segundoNumero

if __name__ == '__main__':
  objeto = Calculadora(10,7)
  print(objeto.soma())
```

Ponto de vista funcional

    . Adição de quaisquer sequência de números
    . Elementos que constituem a operação
    . Operação em si

```python
def soma(a,b):
  soma = a + b
  return soma
soma(10,7)
```

Principais linguagens de programação

    .Lua: wordwarcraft, angry birds, civilization, street figther IV
    .Elixir: Globosat, pinterest, Gopro
    .Scala: Tumbler, Linkedin, Simiens, Twitter

```python

from PIL import Image
Image.open('/content/LP_Funcional.png')
```

      Mas e essa tal de independência?

        1. Possuir ao menos um parâmetro: Irá permitir que o usuário interaja com a aplicação
        2. Deve retornar um objeto (valor, string, função)
        3. Não deve possuir loops: Elementos do paradigma imperativo

        # Usar o minímo necessário de elementos do paradigma imperativo

```python
def criaArray():
  palavra = 'olá mundo'
  lista = []
  for i in palavra:
    lista+=[i]

  print(lista)

criaArray()
```

    Função de alta ordem é uma função que recebe ou retorna uma função.
    Só é a possível definir uma função de alta ordem, se a linguagem tem
    funções de primeira classe, pois é necessário que funções sejam "passáveis".

```python
def criaArray(palavra):
  f = lambda i:i
  return list(map(str, f(palavra)))
criaArray('olá mundo')
```

      Caracteristicas básicas de uma função

        . Imutável
          - Criar novos objetos, ao invés de ficar operando sobre um mesmo
        . Sem mudança de estados
          - Evitar efeitos colaterais ao longo da operação

Criação de lista sem mutabilidade

```python
def separa():
  array = list()
  y = 0

  for i in range(1,7+1):

    valores = int(input(f"Digite o {i}° valor: "))

    if valores % 2 == 0:
      y+=1
      array.insert(0,valores)
    else:
      array.insert(len(array),valores)

  print(sorted(array[:y]) + sorted(array[y:]))

separa()
```

    Mesma operação com mutabilidade:

    . Criar uma lista
    . Atruibuir a lista como parametro
    . Interagir com o parametro
    . Criar uma nova lista

    # Função de primeira classe

```python
def ordena(lista):
  novaLista = sorted(filter(lambda i: i % 2 == 0, lista)) + sorted(filter(lambda i: i % 2 == 1, lista))
  return novaLista

lista = [int(input(f'{c+1}° número: ')) for c in range(7)]
ordena(lista)
```

    Ganho:
    . Torna os programas mais próximos de expressões matemáticas
    . Torna as declarações mais simples e direta
    . Confinamento de valores em espaços imutáveis na memória
    . Torna o código mais amigável a leitura e refatoramento

#Efeitos colaterais

```python
from datetime import date
def atribui():
  data_atual = date.today()
  nova_data = str(data_atual)
  data_final = int(nova_data[5:7]) + 1
  return data_final
atribui()
```

```python
import requests
from time import sleep
def resposta(request):
  sleep(5)
  return request.status_code
resposta(requests.get('https://github.com/'))
```

    Elementos da programação funcional:
     - lambda
     - filter
     - map
     - reduce

```python
def nome(parametro):
  return expressão
```

```python
nome = lambda parametro:expressao
```

```python
def f(x):
  return x**2
f(2)
```

```python
f = lambda i: i**2
f(2)
```

```python
def interacao(lista):
    g = lambda x: (x**3 - 3*x + 2)**((-x/4) - 1)
    return list(map(g, itens))
ret([2,3,2,3,2,3])
```

```python
Image.open('/content/cerebro.jpeg')
```

```python
Image.open('/content/filtro.jpg')
```

```python
from sympy import *
```

```python
f = lambda x: ((x**3 - 3*x+2)**((-x/4)-(1)))
f(5)
```

```python
var('x')
f = lambda x: diff(x)
f((x**2)/(x**3))
```

```python
def analiseCurva():
  # Derivada de primeira ordem
  var('x')
  func = ((1/3)*(x**3)- ((5/2)*(x**2)) + 6*x)
  f = lambda x: diff(x)
  # Passar a equação no domino
  dominio = [-2,-1,0,1, 2, 3]
  g = list(map(lambda dominio: (dominio**2 - 5*dominio + 6), dominio))

  # Classificação da curva
  resultado = []
  for c in g:
    if c == 0:
      resultado.append('anula')
    elif c > 0:
      resultado.append('crescente')
    else:
      resultado.append('decrescente')

  #Resultados
  print(Symbol(str(f(func))))
  print(dominio)
  print(resultado)

analiseCurva()
```

    Foto da análise da curva

```python
from PIL import Image
Image.open('')
```

```python
var('x')
f = lambda x: integrate(x)
f(x**2)
```

    Aplicação estatística:

    - Correlação linear simples
    - Regressão linear simples

    Os estudos ligados a regressão aplicam-se aquelas situações
    em que há razões para supor uma relação de causa-efeito
    entre duas variáveis quatitativas e se deseja expressar
    matematicamente essa relação

    Relacionamento entre X (variável independente, explicativa) e
    Y (variável dependente, resposta)

    Esse relacionamento é explicado por um modelo matemático,
    uma equação que associa a variável dependente com a independente

    Ex: Resultados para o teor de cálcio no solo (x) e a porcentagem
    de turberculos maduros (y) em uma parcela aleatória da população

```python
from functools import reduce
import numpy as np
from statistics import mean
def RegressaoLinear(calcio_x, tuberculos_y):

  #Pesos
  n = len(calcio_x)
  soma_x = reduce(lambda calcio_x, i: i+calcio_x, calcio_x)
  soma_y = reduce(lambda tuberculos_y, y: y+tuberculos_y, tuberculos_y)
  soma_xy = sum(x*y for x,y in zip(calcio_x,tuberculos_y))
  soma_quadrada_x = sum(np.array(calcio_x)**2)
  soma_quadrada_y = sum(np.array(tuberculos_y)**2)

  #Estatistica do teste
  #Coeficiente de correlaçao de Person
  R2 = ((n*soma_xy)-(soma_x*soma_y))/sqrt(((n*soma_quadrada_x)-soma_x**2)*((n*soma_quadrada_y)-(soma_y**2)))
  t = R2*sqrt((n-2)/(1 - R2*R2))

  tStudent = [{
    '7': [12.7062,4.3027,3.1824,2.7765,2.5706,2.4469,2.3646]
  }]
  ts = tStudent[0]['7'][6]

  #H0: Não existe correlação
  #H1: Existe correlação

  if abs(t) >=ts:
    print(' Rejeita-se H0\n',
  f'Com o valor t = {t:.2f} sendo maior que o valor tabelado {ts}\n',
  'rejeitamos H0 e concluimos que há correlação entre o teor de cálcio no solo\n',
  f'e a porcentagem de tuberculos maduros de acordo com a classificação r = {R2:.2f}\n')
  else:
    print('Rejeita-se H1')

  #Construindo a reta de regressão
  media_x = mean(np.array(calcio_x))
  media_y = mean(np.array(tuberculos_y))

  #Construindo a reta estimada de regresaão: Y = a +bx
  beta = (soma_xy - n * media_x * media_y ) / (soma_quadrada_x - n * ( media_x * media_x ) )
  alfa = ((media_y)-(beta*media_x))

  #Predição
  predicao = lambda x: alfa + (beta*x)
  num = float(input(' Qual o valor de x deseja predizer?: '))
  print(f' Com a reta real sendo, y = {alfa:.2f} + {beta:.2f}X')
  print(f' Para x = {num}, temos y = {predicao(1.1)}')

RegressaoLinear([0.2, 0.3, 0.4, 0.5, 0.7, 0.8, 1.0, 1.1, 1.3],[75, 79, 80, 86, 88, 89, 93, 95, 99])
```


# Physics 256
## Spectral Analysis

If you are not familiar with Fourier analysis, I strongly reccomend the following video: https://www.youtube.com/watch?v=spUNpyF58BY

```python
import style
style._set_css_style('../include/bootstrap.css')

from IPython.display import YouTubeVideo
YouTubeVideo('spUNpyF58BY')
```

## Last Time

### [Notebook Link: 20 Least Squares](./20_LeastSquares.ipynb)

- Non-linear fitting
- Goodness of fit and $\chi^2$

## Today

- Spectral analysis
- Discrete Fourier transform

## Setting up the Notebook

```python
import matplotlib.pyplot as plt
import numpy as np
%matplotlib inline
plt.style.use('../include/notebook.mplstyle');
%config InlineBackend.figure_format = 'svg'
colors = plt.rcParams['axes.prop_cycle'].by_key()['color']
```

## Spectral Analysis

There are entire courses taught on this subject (EE 171) where it is often referred to as *signal processing*.  We will learn how to extract information about the predictable or repeating parts of signals.

For a continous function $y(t)$, this is achieved by taking the Fourier transform:

\begin{equation}
\hat{y}(\omega) = \int_{-\infty}^{\infty} dt y(t) \mathrm{e}^{-i \omega t}.
\end{equation}

What do we do when we have discrete data sets?

### Discrete Fourier Transform

Consider a vector of $N$ data points $\vec{y} = (y_0,\ldots,y_{N-1})$.  This will often be a *time series*, the value of a function over a long period of time.  We assume that the measurements $y_j$ are made at equally spaced intervals where $t_j = j \Delta t$ with $j \in \mathbb{Z}$.

In order to search for any repeating patterns in $\vec{y}$ we can represent construct a continuous represenation $y(t)$ using the *Dirac comb*:

\begin{equation}
y(t) = \sum_{j=0}^{N-1} y_j \delta(t - j\Delta t)
\end{equation}

then using our definition for the continuous Fourier transform:

\begin{align}
\hat{y}(\omega) &= \int_{-\infty}^{\infty} dt y(t) \mathrm{e}^{-i \omega j\Delta t} \newline
&= \sum_{j=0}^{N-1} y_j \mathrm{e}^{-i \omega t} .
\end{align}

However, unlike the continous case, our signal has a finite range, $T$.  We can extend it by enforcing that $y(t+T) = y(t)$, i.e. we have a time-translational symmetry by $T$. This implies $y_j = y_{j+N}$ and

\begin{align}
\sum_{j=0}^{N-1} y_j \mathrm{e}^{-i \omega j \Delta t}  &= \sum_{j=0}^{N-1} y_{j+N} \mathrm{e}^{-i \omega (j+N)\Delta t} \newline
&= \sum_{j=0}^{N-1} y_{j} \mathrm{e}^{-i \omega j\Delta t} \mathrm{e}^{-i \omega T}
\end{align}

which is only possible if:

\begin{equation}
\mathrm{e}^{-i \omega T} = 1 \Leftrightarrow \omega = \omega_k =  \frac{2\pi}{N\Delta t} k \text{ where } k \in \mathbb{Z}.
\end{equation}

1. The smallest non-zero frequency is: $$\omega_1 = \frac{2\pi}{N\Delta t}.$$  We need lots of data at long times to investigate small frequencies.
2. The largest frequency we can study is $$\omega_{N-1} = \frac{2\pi(N-1)}{N\Delta t} \approx \frac{2\pi}{\Delta t}.$$  To study high frequency behavior we need a small time step $\Delta t \ll 1$.

We can now write the discrete Fourier transform of $y$ as:

\begin{align}
\hat{y}_k & = \sum_{j=0}^{N-1} y_j \mathrm{e}^{-i \omega_k j \Delta t} \newline
& = \sum_{j=0}^{N-1} y_j \mathrm{e}^{-i 2\pi k j /N} .
\end{align}

Note:
\begin{equation}
\hat{y}_{k+N} = \sum_{j=0}^{N-1} y_j \mathrm{e}^{-i 2\pi (k+N) j /N} = \sum_{j=0}^{N-1} y_j \mathrm{e}^{-i 2\pi j k/N}\mathrm{e}^{-i 2\pi j}= \hat{y}_k.
\end{equation}
There are only $N$ values of $k$.

### Inverse Fourier Transform

To reconstruct our original data set, we multiply both sides by $\mathrm{e}^{i 2\pi n k/N}$ and sum over $k$:

\begin{align}
\sum_{k=0}^{N-1} \hat{y}_k \mathrm{e}^{i 2\pi n k/N} & = \sum_{k=0}^{N-1}\mathrm{e}^{i 2\pi n k/N} \sum_{j=0}^{N-1} y_j \mathrm{e}^{-i 2\pi k j /N} \newline
 &= \sum_{j=0}^{N-1} y_j \sum_{k=0}^{N-1} \mathrm{e}^{-i 2\pi k(j-n)/N}
\end{align}

and we recognize the definition of the Kronecker $\delta$-function:
\begin{equation}
\sum_{k=0}^{N-1} \mathrm{e}^{-i 2\pi k(j-j')/N} = N \delta_{j,j'}
\end{equation}
such that
\begin{equation}
y_n = \frac{1}{N} \sum_{k=0}^{N-1}\hat{y}_k \mathrm{e}^{i 2\pi n k/N} .
\end{equation}

<div class="span alert alert-success">
<h2> Programming challenge </h2>
Complete the code fragment below to implement the discrete Fourier transform.
</div>

```python
from scipy.constants import pi as π
def fourier_transform(y):
    '''Return the discrete Fourier transform of y.'''
    N = y.size
    ŷ = np.zeros([N],dtype=complex)

    return ŷ
```

```python
# the time series
Δt = 0.1
t = np.arange(0.0,50.0,Δt)
N = t.size
ωₒ = 2.0*π*(0.2)
ϕ = 0.5*π
y = np.sin(ωₒ*t + ϕ) + np.sin(2*ωₒ*t)

# Plot the time series
fig,ax = plt.subplots(1,2, figsize=(8,4))
fig.subplots_adjust(wspace=0.4)

ax[0].plot(t,y)
ax[0].set_xlabel('Time [s]')
ax[0].set_ylabel('Amplitude [arb.]')

# Get the discrete FT
ω = 2*π*np.arange(0,N)/(N*Δt)
ŷ = fourier_transform(y)

# Plot the real and imaginary parts of the DFT
ax[1].plot(ω,ŷ.real,label='real.', color=colors[0])
ax[1].plot(ω,ŷ.imag,linestyle='--',label='imag.', color=colors[-2])

ax[1].set_xlabel('Frequency [rad/s]')
ax[1].set_ylabel('Transform [arb.]')
ax[1].legend(loc='lower right')
```

## Power Spectrum

Sometimes we don't want the additional phase information and we only care about the location of the peaks in the spectrum.  It is related to the integrated temporal correlation function of the time series.  For a continuous function:

\begin{equation}
\int_{-\infty}^{\infty} d\tau \left[\int_{-\infty}^{\infty} dt y(t)y^\ast(t+\tau) \right] \mathrm{e}^{i\omega \tau} = \hat{y}(\omega)\hat{y}^\ast(\omega) = |\hat{y}(\omega)|^2
\end{equation}

where we have used the convolution property of the Fourier transform.

We can analze it for our previous time series in the discrete case:

```python
plt.plot(ω,np.abs(ŷ),color=colors[2])
plt.xlabel('Frequency [rad/s]')
plt.ylabel('Power [arb]')
```

### Let's investigate the power of the Fourier transform on some input data

```python
t,*y = np.loadtxt('../data/signal.dat',unpack=True)
fig, axes = plt.subplots(1,2,sharex=True, sharey=True, squeeze=True, figsize=(8,4))
fig.subplots_adjust(wspace=0.1)

for i,ax in enumerate(axes):
    ax.plot(t,y[i],'-',color=colors[i], linewidth=1)

axes[0].set_ylabel('y')
[ax.set_xlabel('t') for ax in axes]
```

```python
ω = 2*π*np.arange(0,t.size)/(t.size*(t[1]-t[0]))

fig, axes = plt.subplots(1,2,sharex=True, sharey=True, squeeze=True, figsize=(8,4))
fig.subplots_adjust(wspace=0.1)

for i,ax in enumerate(axes):
    ŷ = fourier_transform(y[i])
    ax.plot(ω,np.abs(ŷ),color=colors[i], linewidth=1)
    ax.set_xlabel('ω')
    ax.set_xlim(0,ω[-1])

axes[0].set_ylabel('|ŷ|')
```

In all these examples, we have identifed some *natural* frequecies in the time series, but there were always two copies.

### What is the origin of the large frequency signals?

For a real signal we have: $y_j = y^\ast_j$.  Consider the complex conjugate of the Fourier tranform $\hat{y}^\ast_k$:

\begin{align}
\hat{y}^\ast_k &= \sum_{j=0}^{N-1} y^\ast_j \mathrm{e}^{i2\pi jk/N} \newline
 &= \sum_{j=0}^{N-1} y_j \mathrm{e}^{-i2\pi j(-k)/N} \times \mathrm{e}^{-i 2\pi j N/N} \newline
 &= \sum_{j=0}^{N-1} y_j \mathrm{e}^{-i2\pi j(N-k)/N}  \newline
 &= \hat{y}_{N-k} .
\end{align}

Thus, the $N$ real data points are used to construct $N/2$ unique *complex* data.  The spectrum is completely described by the first $N/2$ frequencies.  Instead of $2\pi/\Delta t$ the largest frequency we can distinguish is half this value.

### Nyquist Frequency
$$\omega_{\rm Ny} = \frac{\pi}{\Delta t}$$

We can simply truncate our discrete Fourier transform, plotting the meaningful independent components.

```python
fig, axes = plt.subplots(1,2,sharex=True, sharey=True, squeeze=True, figsize=(8,4))
fig.subplots_adjust(wspace=0.1)

for i,ax in enumerate(axes):
    ŷ = fourier_transform(y[i])
    ax.plot(ω,np.abs(ŷ),color=colors[i], linewidth=1)
    ax.set_xlabel('ω')
    ax.set_xlim(0,0.5*ω[-1])

axes[0].set_ylabel('|ŷ|')
```

In practice, we want $\omega_{\rm Ny}$ to be considerably larger then any Fourier components that are expected to be present in the signal.

1. In experiments: use a low-pass filter.
2. In simulations, make sure $\Delta t \ll$ any intrinsic time scales.

```python

```


Runge-Kutta methods
===================

```
%matplotlib inline
```

```
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
from matplotlib import rcParams
rcParams['font.family'] = 'serif'
rcParams['font.size'] = 16
rcParams['figure.figsize'] = (12,6)
```

Considering IVPs of the form

$${\boldsymbol{y}}'(x) = {\boldsymbol{f}}(x, {\boldsymbol{y}}(x)).$$

We looked at multistage (e.g. Runge-Kutta) methods. These

-   self start,

-   compute next iteration by combining estimates of ${\boldsymbol{f}}$

to improve order of accuracy.

Methods constructed by matching terms in Taylor expansion; ensures local
truncation error is a certain order. May improve efficiency by checking
if results are “good enough” at given $h$.

### Comparing errors

Use $y_e(x_0 + h)$ as the *exact* solution to

$$y'(x) = f(x, y(x)), \quad y(x_0) = y_0.$$

Consider numerical solution $y_h(x_0 + h)$ for one step (many stages) of
a multistage method. By construction the Taylor expansion implies

$$| y_e(x_0 + h) - y_h(x_0 + h) | = C h^{s+1}.$$

Know that method has order $s$; to compute error also
need $C$.

Can use Richardson extrapolation variant: compare the error with two
different step sizes.

By assumption taking two steps of size $h/2$ gives

$$| y_e(x_0 + h) - y_{h/2}(x_0 + h) | = 2 C (h/2)^{s+1}.$$

Rearranging gives

$$\text{Local truncation error} = C h^{s+1} = \frac{y_h - y_{h/2}}{1 - 2^{-s}}.$$

### Adaptive stepping

Given a computable estimate of the error, decide whether our result is
sufficiently accurate automatically. As with adaptive quadrature, can
modify $h$ where necessary.

Two key differences between algorithms for quadrature and IVPs.

1.  IVP step depends on previous step: $h_1$ at $x_1$ different from $h_0$ at $x_0$.

    Expect accuracy depends on local properties ($\lambda \sim
    |\partial_y f|$); always use latest $h_1$, not initial $h_0$.

    Must *increase* $h$ when accuracy sufficient.

2.  Richardson extrapolation expensive (3 times original algorithm).

Instead combine estimates in different ways to estimate error
(e.g. RKF45: 6 evaluations, $4^{\text{th}}$ order update and
$5^{\text{th}}$ order error check).

RKF45 seems optimal for number of function evaluations:

| Number of function evaluations | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 |
|:------------------------------:|---|---|---|---|---|---|---|---|
| Maximum order of R-K method    | 1 | 2 | 3 | 4 | 4 | 5 | 6 | 6 |

Multistep Methods
=================

Loosely a multistage method approximates $y'$ in

$$y'(x) = f(x, y(x)), \quad y(x_0) = y_0.$$

Loosely a *multistep* method approximates the quadrature solution

$$y_{n+1} - y_n = \int_{x_n}^{x_{n+1}} f(x, y(x)) \, d x.$$

Difference comes from interpretation and assumption: that the quadrature
can be well approximated using a formula depending on $y_{n-j}$ (and
$x_{n-j}$). This gives the general formula

$$a_k y_{n+1} + a_{k-1} y_n + \dots + a_0 y_{n+1-k} = h \left[ b_k f_{n+1} + b_{k-1} f_n + \dots + b_0 f_{n+1-k} \right].$$

### $k$-step methods

In a *$k$-step method*

$$a_k y_{n+1} + a_{k-1} y_n + \dots + a_0 y_{n+1-k} = h \left[ b_k f_{n+1} + b_{k-1} f_n + \dots + b_0 f_{n+1-k} \right]$$

all terms on the left (except $y_{n+1}$) are known, and all terms on the
right (except $f_{n+1}$) are computable. The coefficients $a_m,
  b_m$ fix the method.

To give $y_{n+1}$ (essential!) must have $a_k \neq 0$.

If $b_k = 0$ can directly compute $y_{n+1}$; the method is *explicit*.

If $b_k \neq 0$ then required $f_{n+1}$ is not computable; the method is
*implicit*. Either

1.  use iteration to solve nonlinear equation, *or*

2.  use predictor-corrector method (usual approach).

Adams-Bashforth methods
-----------------------

The Adams-Bashforth methods are explicit $k$-step methods with formula

$$y_{n+1} - y_n = h \left[ b_{k-1} f_n + b_{k-2} f_{n-1} + \dots \right]$$

based on evenly spaced points $x_j$.

To compute the coefficients, require that the method is “as accurate as
possible”; i.e., that it is exact for as many polynomials as possible.

### Adams-Bashforth methods: example

For example, the 2-step method

$$y_{n+1} - y_n = h \left[ b_1 f_n + b_0 f_{n-1} \right]$$

should be exact for polynomials of degree 0, 1. Approximate

\begin{align}
  y_{n+1} - y_n & = \int_{x_n}^{x_{n+1}} f(x, y(x)) \, dx \\
  & \simeq h \left[ b_1 f_n + b_0 f_{n-1} \right].
\end{align}

Insist the formula holds for $f(x) = p_m(x) = x^m$ for $m = 0, 1$.

Without loss of generality choose $x_n = 0$, $x_{n-1} = -h$:

\begin{align}
  p_0 & = 1: & h & = h \left[ b_1 + b_0 \right] \\
  p_1 & = x: & h^2 / 2 & = h \left[ -h b_0 \right].
\end{align}

Hence $b_0 = -1 / 2$, $b_1 = 3 / 2$.

### Standard example

Apply the 2 step Adams-Bashforth method to

$$y'(x) = - \sin(x), \quad y(0) = 1$$

and integrate to $x = 0.5$. Use Euler predictor-corrector to start
algorithm. Using $h = 0.1$ gives

|$n$|$x_n$|$y_n$|$f(x_n, y_n)$|$\cos(x_n)$|
|---|-----|-----|-------------|-----------|
|0  | 0.0 |1.000| 0.000       | 1.000     |
|1  | 0.1 |0.995| -0.100      | 0.995     |
|2  | 0.2 |0.980| -0.199      | 0.980     |
|3  | 0.3 |0.955| -0.296      | 0.955     |
|4  | 0.4 |0.921| -0.389      | 0.921     |
|5  | 0.5 |0.877|             | 0.878     |

The error is $0.4\%$, just visible at this precision. With $h =
  0.01$ the error is $0.006\%$. Convergence is nearly second order;
result is slightly biased by the starting method.

```
def ab2_method(f, x_end, y0, N):
    """Solve IVP y'=f(x, y(x)) on x \in [0, x_end] with y(0) = y0 using N+1 points, using 2nd order Adams-Bashforth method."""

    h = x_end / N
    x = np.linspace(0.0, x_end, N+1)

    y = np.zeros((N+1, len(y0)))
    fn = np.zeros_like(y)
    y[0, :] = y0

    # Start using Euler predictor-corrector step
    fn[0, :] = f(x[0], y[0, :])
    yp = y[0, :] + h * fn[0, :]
    y[1, :] = y[0, :] + h / 2.0 * (fn[0, :] + f(x[1], yp))

    for n in range(1, N):
        fn[n, :] = f(x[n], y[n, :])
        y[n+1, :] = y[n, :] + h / 2.0 * (3.0 * fn[n, :] - fn[n-1, :])

    return x, y

def fn_simple(x, y):
    """Simple function for example"""

    return -np.sin(x)

# Example given above
x_5, y_5 = ab2_method(fn_simple, 0.5, [1.0], 5)
x_50, y_50 = ab2_method(fn_simple, 0.5, [1.0], 50)

print("Solution at x = 0.5 using h = 0.1 is y = {}.".format(y_5[-1, 0]))
print("Solution at x = 0.5 using h = 0.01 is y = {}.".format(y_50[-1, 0]))
```

    Solution at x = 0.5 using h = 0.1 is y = 0.8771932824110381.
    Solution at x = 0.5 using h = 0.01 is y = 0.8775775811228846.

```
Npoints = np.array([2**i for i in range(3, 12)])
h = 0.5 / Npoints
err = np.zeros_like(h)

for i in range(len(Npoints)):
    N = Npoints[i]
    x, y = ab2_method(fn_simple, 0.5, [1.0], N)
    err[i] = np.abs(np.cos(0.5) - y[-1, 0])

# Find the best fit to the data: if err ~ A h^s then the best fit straight line of log(err) vs log(h) has slope s.
p = np.polyfit(np.log(h[1:]), np.log(err[1:]), 1)

fig = plt.figure(figsize = (10,6))
ax = fig.add_subplot(111)
ax.loglog(h, err, 'kx', label = 'Numerical data')
ax.loglog(h, np.exp(p[1])*h**(p[0]), 'b-', label = "Line slope {:.2f}".format(p[0]))
ax.set_xlabel('h')
ax.set_ylabel('$\|$ Error $\|$')
ax.set_title("Convergence of the $2^{nd}$ order Adams-Bashforth method for $y' = -\sin(x)$")
ax.legend(loc = 2)
fig.tight_layout()
plt.show()
```

### Standard Example: 2

Consider the system

$$\left\{
\begin{aligned}
  \dot{x} & = -y \\ \dot{y} & = x
\end{aligned} \right., \quad x(0) = 1, \, \, y(0) = 0.$$

In polar coordinates this is $\dot{r} = 0$, $\dot{\phi} = 1$.

Use Adams-Bashforth 2 step method with $h=0.1$. At
$t=500$ the result matches the correct answer to the eye.
The growth of the radius makes the errors visible.

Use Adams-Bashforth 2 step method with $h=0.01$. At
$t=500$ the result matches the correct answer to the eye.

Looking at the growth of the radius makes the errors visible even now.

```
def fn_circular_motion(x, y):
    """Simple function for Euler's method example"""

    dfdt = np.zeros_like(y)
    dfdt[0] = -y[1]
    dfdt[1] =  y[0]

    return dfdt

# Example given above
y0 = np.array([1.0, 0.0])
t_0_1, y_0_1 = ab2_method(fn_circular_motion, 500.0, y0, 5000)
t_0_01, y_0_01 = ab2_method(fn_circular_motion, 500.0, y0, 50000)

fig = plt.figure(figsize = (10,10))
ax1 = fig.add_subplot(221)
ax1.plot(y_0_1[:, 0], y_0_1[:, 1], 'b-', label = '$h = 0.1$')
ax1.set_xlabel('$x$')
ax1.set_ylabel('$y$')
ax2 = fig.add_subplot(222)
ax2.plot(y_0_01[:, 0], y_0_01[:, 1], 'b-', label = '$h = 0.01$')
ax2.set_xlabel('$x$')
ax2.set_ylabel('$y$')
ax3 = fig.add_subplot(223)
ax3.plot(t_0_1, np.abs(y_0_1[:, 0]**2 + y_0_1[:, 1]**2 - 1.0), 'b-', label = '$h = 0.1$')
ax3.set_xlabel('$t$')
ax3.set_ylabel('Growth in $r$')
ax4 = fig.add_subplot(224)
ax4.plot(t_0_01, np.abs(y_0_01[:, 0]**2 + y_0_01[:, 1]**2 - 1.0), 'b-', label = '$h = 0.01$')
ax4.set_xlabel('$t$')
ax4.set_ylabel('Growth in $r$')
fig.tight_layout()
plt.show()
```

### Higher order Adams-Bashforth

Principle behind higher order Adams-Bashforth methods the same; solve
for the coefficients by integrating

$$y_{n+1} - y_n = h \left[ b_k f_n + b_{k-1} f_{n-1} + \dots b_0
f_{n + 1 - k} \right]$$

over the range, assuming $f(x) = p_s(x)$ for polynomials of degree
$s = 0, \dots k$.

Easiest to choose as a basis for the polynomials

$$p_{s+1}(x) = x (x + h) \dots (x + h s)$$

as $p_{s+1}$ vanishes at $x = 0, -h, \dots, -h s$. Then choose
$x_n = 0, x_{n - 1} = -h$ and so on: the resulting system of equations
is in upper-triangular form – solve by back substitution.

### Higher order Adams-Bashforth example

For example, the Adams-Bashforth method of order five has formula

$$y_{n+1} - y_n = h \left[ b_4 f_n + b_3 f_{n-1} + b_2 f_{n-2} + b_1 f_{n-3} + b_0 f_{n-4} \right] ,$$

and the coefficients follow from

\begin{align}
p_0 & = 1: & h & = h \left[ b_4 + b_3 + b_2 + b_1 + b_0 \right] \\
p_1 & = x: & \frac{h^2}{2} & = h \left[ -h \left( b_3 + 2 b_2 + 3
    b_1 + 4 b_0 \right) \right] \\
p_2 & = x (x + h): & 5 \frac{h^3}{6} & = h \left[ h^2 \left( 2 b_2
    + 6 b_1 + 12 b_0 \right) \right] \\
p_3 & = x (x + h) (x + 2 h): & 9 \frac{h^4}{4} & = h \left[ -h^3
  \left( 6 b_1 + 24 b_0 \right) \right] \\
p_4 & = x (x + h) (x + 2 h) (x + 3 h): & 251 \frac{h^5}{30} & = h
\left[ 24 h^4 b_0 \right].
\end{align}

### Standard example

Apply the 5 step Adams-Bashforth method to

$$y'(x) = - \sin(x), \quad y(0) = 1$$

and integrate to $x = 0.5$. Use RK4 method to start algorithm. Using
$h = 0.05$ gives

|$n$|$x_n$|$y_n$|$f(x_n, y_n)$|$\cos(x_n)$|
|---|-----|-----|-------------|-----------|
|0 | 0.0 | 1.000 | 0.000 | 1.000|
|2 | 0.1 | 0.995 | -0.100 | 0.995|
|4 | 0.2 | 0.980 | -0.199 | 0.980|
|6 | 0.3 | 0.955 | -0.296 | 0.955|
|8 | 0.4 | 0.921 | -0.389 | 0.921|
|10 |0.5 |0.878| |0.878|

The error is $3 \times 10^{-6}\%$, not visible at this precision. With
$h = 0.01$ the error is $2 \times 10^{-9}\%$. Convergence is slightly
worse than fifth order; result is slightly biased by the starting
method.

```
def ab5_method(f, x_end, y0, N):
    """Solve IVP y'=f(x, y(x)) on x \in [0, x_end] with y(0) = y0 using N+1 points, using 5th order Adams-Bashforth method."""

    h = x_end / N
    x = np.linspace(0.0, x_end, N+1)

    y = np.zeros((N+1, len(y0)))
    fn = np.zeros_like(y)
    y[0, :] = y0

    # Start using RK4 steps
    for n in range(4):
        fn[n, :] = f(x[n], y[n, :])
        k1 = h * fn[n, :]
        k2 = h * f(x[n] + h / 2.0, y[n, :] + k1 / 2.0)
        k3 = h * f(x[n] + h / 2.0, y[n, :] + k2 / 2.0)
        k4 = h * f(x[n] + h, y[n, :] + k3)
        y[n+1, :] = y[n, :] +  (k1 + k4 + 2.0 * (k2 + k3)) / 6.0

    # Solve for the coefficients

    coeff_A = np.array([[1.0, 1.0, 1.0, 1.0, 1.0],[0.0, -1.0, -2.0, -3.0, -4.0],[0.0, 0.0, 2.0, 6.0, 12.0],
                        [0.0, 0.0, 0.0, -6.0, -24.0],[0.0, 0.0, 0.0, 0.0, 24.0]])
    coeff_b = np.array([1.0, 1.0 / 2.0, 5.0/6.0, 9.0/4.0, 251.0/30.0])
    b = np.linalg.solve(coeff_A, coeff_b)

    for n in range(4, N):
        fn[n, :] = f(x[n], y[n, :])
        y[n+1, :] = y[n, :] + h * (b[0]*fn[n, :] + b[1]*fn[n-1, :] + b[2]*fn[n-2, :] + b[3]*fn[n-3, :] + b[4]*fn[n-4, :])

    return x, y

def fn_simple(x, y):
    """Simple function for example"""

    return -np.sin(x)

# Example given above
x_5, y_5 = ab5_method(fn_simple, 0.5, [1.0], 5)
x_50, y_50 = ab5_method(fn_simple, 0.5, [1.0], 50)

print("Solution at x = 0.5 using h = 0.1 is y = {}.".format(y_5[-1, 0]))
print("Error at x = 0.5 using h = 0.1 is y = {}.".format(abs(np.cos(0.5) - y_5[-1, 0])))
print("Solution at x = 0.5 using h = 0.01 is y = {}.".format(y_50[-1, 0]))
print("Error at x = 0.5 using h = 0.01 is y = {}.".format(abs(np.cos(0.5) - y_50[-1, 0])))
```

    Solution at x = 0.5 using h = 0.1 is y = 0.8775828784830745.
    Error at x = 0.5 using h = 0.1 is y = 3.165927017212411e-07.
    Solution at x = 0.5 using h = 0.01 is y = 0.8775825619049437.
    Error at x = 0.5 using h = 0.01 is y = 1.4570900042087942e-11.

### Standard Example: 2

Consider the system

$$
\left\{
\begin{aligned}
  \dot{x} & = -y \\ \dot{y} & = x
\end{aligned} \right., \quad x(0) = 1, \, \, y(0) = 0.
$$

In polar coordinates this is $\dot{r} = 0$, $\dot{\phi} = 1$.

Use the Adams-Bashforth 5 step method with $h=0.1$. At
$t=500$ the result matches the correct answer to the eye.

Looking at the growth of the radius makes the errors visible.

Use Adams-Bashforth 5 step method with $h=0.01$. At
$t=500$ the result matches the correct answer to the eye.

Looking at the growth of the radius makes the errors visible even now.

```
def fn_circular_motion(x, y):
    """Simple function for circular motion example"""

    dfdt = np.zeros_like(y)
    dfdt[0] = -y[1]
    dfdt[1] =  y[0]

    return dfdt

# Example given above
y0 = np.array([1.0, 0.0])
t_0_1, y_0_1 = ab5_method(fn_circular_motion, 50.0, y0, 500)
t_0_01, y_0_01 = ab5_method(fn_circular_motion, 50.0, y0, 5000)

fig = plt.figure(figsize = (10,6))
ax1 = fig.add_subplot(121)
ax1.plot(y_0_1[:, 0], y_0_1[:, 1], 'b-', label = '$h = 0.1$')
ax1.set_xlabel('$x$')
ax1.set_ylabel('$y$')
ax2 = fig.add_subplot(122)
ax2.plot(y_0_01[:, 0], y_0_01[:, 1], 'b-', label = '$h = 0.01$')
ax2.set_xlabel('$x$')
ax2.set_ylabel('$y$')
fig.tight_layout()
plt.show()
```

```
fig = plt.figure(figsize = (10,6))
ax1 = fig.add_subplot(121)
ax1.plot(t_0_1, np.sqrt(y_0_1[:, 0]**2 + y_0_1[:, 1]**2)-1.0, 'b-', label = '$h = 0.1$')
ax1.set_xlabel('$t$')
ax1.set_ylabel('$r - 1$')
ax2 = fig.add_subplot(122)
ax2.plot(t_0_01, np.sqrt(y_0_01[:, 0]**2 + y_0_01[:, 1]**2)-1.0, 'b-', label = '$h = 0.01$')
ax2.set_xlabel('$t$')
ax2.set_ylabel('$r - 1$')
fig.tight_layout()
plt.show()
```

```
Npoints = np.array([50*2**i for i in range(2, 10)])
h = np.zeros_like(Npoints, float)
h = 50.0 / Npoints
err = np.zeros_like(h)

for i in range(len(Npoints)):
    N = Npoints[i]
    x, y = ab5_method(fn_circular_motion, 50.0, y0, N)
    err[i] = np.abs(y[-1, 0]**2 + y[-1, 1]**2 - 1.0)

# Find the best fit to the data: if err ~ A h^s then the best fit straight line of log(err) vs log(h) has slope s.
p = np.polyfit(np.log(h[1:]), np.log(err[1:]), 1)

fig = plt.figure(figsize = (10,6))
ax = fig.add_subplot(111)
ax.loglog(h, err, 'kx', label = 'Numerical data')
ax.loglog(h, np.exp(p[1])*h**(p[0]), 'b-', label = "Line slope {:.2f}".format(p[0]))
ax.set_xlabel('h')
ax.set_ylabel('$\|$ Error $\|$')
ax.set_title("Convergence of $5^{th}$ order Adams-Bashforth method for circular motion example")
ax.legend(loc = 2)
fig.tight_layout()
plt.show()
```

Summary
=======

-   The Runge-Kutta Fehlberg method, often known as RK45, is a fourth order method that estimates its own error using a fifth order computation.
-   When combined with adaptive step sizes this is a standard method for solving IVPs to moderate accuracy efficiently. The Matlab command is ode45.
-   The multistep methods estimate the quadrature solution of the IVP, not the derivative.
-   Multistep methods use a (given) number of prior values of the solution at each step.
-   Multistep methods require fewer function evaluations than multistage methods such as Runge-Kutta for the same order of accuracy.
-   It is difficult to use adaptive stepping with multistep methods.


```python
import sympy
from sympy import *
print(sympy.__version__)
init_printing(use_unicode=True)
```

    1.1.1

```python
x = Symbol('x', real=True, positive=True)
y = Symbol('y', real=True, positive=True)
z = Symbol('z', real=True, positive=True)
a = Symbol('a', real=True, positive=True)
b = Symbol('b', real=True, positive=True)

y = Eq(a*sinh(b*x))
sols = solve(y, x, rational=False)
print(sols)
```

    []

```python
eul = exp(1)
log(eul)
```


# Intrinsic Camera Calibration with OpenCV

### Goal

The goal of this notebook is to calibrate the intrinsic parameter $K$ of the camera.  Given a number of chessboard images, this notebook first detects the corner positions in each image, and then finds the camera pose of each frame and the camera intrinsic parameters.

* Input:  $n$ chessboard images (= the 2D corner position $x$ and the corresponding 3D position $X$ in the world (chessboard) coordinate system)
* Output: intrinsic parameter $K$, distortion coefficients $d$, $n$ camera poses $R_i, t_i$ w.r.t. the chessboard

To have a better / robust estimation of $K$, the chessboard should be captured as large as possible in the DoF in different poses.

Notice:
* The chessboard coordinate system serves as the world coordinate system.
* Though we obtain $R, t$ as a result, we are not interested in them in this scenario.

### Projection model

Please check the OpenCV document for the detail.
* https://docs.opencv.org/4.0.0/d9/d0c/group__calib3d.html#details

In short, a 3D point $X$ in the world coordinate system (WCS) is transformed to the camera coordinate system (CCS) by a roation $R$ and a translation $t$, and then projected to $x$ by $K$:
\begin{equation}
\tilde{x} \sim K \begin{bmatrix}
R \: | \: t
\end{bmatrix}
\tilde{X}.
\end{equation}

## Libraries

```python
%matplotlib notebook
import sys, os, cv2
import numpy as np
from glob import glob
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D

module_path = os.path.abspath(os.path.join('..'))
if module_path not in sys.path:
    sys.path.append(module_path)

from pycalib.plot import plotCamera

```

## Calibration parameters

```python
# Chessboard configuration
rows = 7   # Number of corners (not cells) in row
cols = 10  # Number of corners (not cells) in column
size = 160 # Physical size of a cell (the distance between neighrboring corners). Any positive number works.

# Input images capturing the chessboard above
input_files = '../data/*.png'

# plotCamera() config
plot_mode   = 0    # 0: fixed camera / moving chessboard,  1: fixed chessboard, moving camera
plot_range  = 4000 # target volume [-plot_range:plot_range]
camera_size = 100  # size of the camera in plot
```

## 3D positions of the chess corners in WCS

```python
X_W = np.empty([rows * cols, 3], dtype=np.float32)
for i_row in range(0, rows):
    for i_col in range(0, cols):
        X_W[i_row*cols+i_col] = np.array([size*i_col, size*i_row, 0], dtype=np.float32)
print(X_W)
```

    [[   0.    0.    0.]
     [ 160.    0.    0.]
     [ 320.    0.    0.]
     [ 480.    0.    0.]
     [ 640.    0.    0.]
     [ 800.    0.    0.]
     [ 960.    0.    0.]
     [1120.    0.    0.]
     [1280.    0.    0.]
     [1440.    0.    0.]
     [   0.  160.    0.]
     [ 160.  160.    0.]
     [ 320.  160.    0.]
     [ 480.  160.    0.]
     [ 640.  160.    0.]
     [ 800.  160.    0.]
     [ 960.  160.    0.]
     [1120.  160.    0.]
     [1280.  160.    0.]
     [1440.  160.    0.]
     [   0.  320.    0.]
     [ 160.  320.    0.]
     [ 320.  320.    0.]
     [ 480.  320.    0.]
     [ 640.  320.    0.]
     [ 800.  320.    0.]
     [ 960.  320.    0.]
     [1120.  320.    0.]
     [1280.  320.    0.]
     [1440.  320.    0.]
     [   0.  480.    0.]
     [ 160.  480.    0.]
     [ 320.  480.    0.]
     [ 480.  480.    0.]
     [ 640.  480.    0.]
     [ 800.  480.    0.]
     [ 960.  480.    0.]
     [1120.  480.    0.]
     [1280.  480.    0.]
     [1440.  480.    0.]
     [   0.  640.    0.]
     [ 160.  640.    0.]
     [ 320.  640.    0.]
     [ 480.  640.    0.]
     [ 640.  640.    0.]
     [ 800.  640.    0.]
     [ 960.  640.    0.]
     [1120.  640.    0.]
     [1280.  640.    0.]
     [1440.  640.    0.]
     [   0.  800.    0.]
     [ 160.  800.    0.]
     [ 320.  800.    0.]
     [ 480.  800.    0.]
     [ 640.  800.    0.]
     [ 800.  800.    0.]
     [ 960.  800.    0.]
     [1120.  800.    0.]
     [1280.  800.    0.]
     [1440.  800.    0.]
     [   0.  960.    0.]
     [ 160.  960.    0.]
     [ 320.  960.    0.]
     [ 480.  960.    0.]
     [ 640.  960.    0.]
     [ 800.  960.    0.]
     [ 960.  960.    0.]
     [1120.  960.    0.]
     [1280.  960.    0.]
     [1440.  960.    0.]]

## 2D positions of the chess corners

```python
Xs_W = []
xs_I = []

for image_path in glob(input_files): # for each chessboard image
    image = cv2.imread(image_path, cv2.IMREAD_GRAYSCALE)      # load the image
    found, x_I = cv2.findChessboardCorners(image, (cols, rows)) # detech the chess corners

    if found: # if found
        term = (cv2.TERM_CRITERIA_EPS+cv2.TERM_CRITERIA_COUNT, 30, 0.1)
        x_I_sub = cv2.cornerSubPix(image, x_I, (5,5), (-1,-1), term) # refine the corner positions
        Xs_W.append(X_W)     # the chess corner in 3D
        xs_I.append(x_I_sub) # is projected to this 2D position
    else:     # if not found
        print("Detection failed :", image_path)
        continue

```

## Calibration

```python
rep, K, d, rvec, tvec = cv2.calibrateCamera(Xs_W, xs_I, (image.shape[1], image.shape[0]), None, None, flags=cv2.CALIB_FIX_ASPECT_RATIO)

print("Intrinsic parameter K = ", K)
print("Distortion parameters d = (k1, k2, p1, p2, k3) = ", d)

```

    Intrinsic parameter K =  [[1.32931637e+03 0.00000000e+00 9.57857318e+02]
     [0.00000000e+00 1.32931637e+03 5.47353253e+02]
     [0.00000000e+00 0.00000000e+00 1.00000000e+00]]
    Distortion parameters d = (k1, k2, p1, p2, k3) =  [[ 0.0052289  -0.01161532  0.0029297   0.00017352  0.0003208 ]]

## 3D Plot

```python
fig_in = plt.figure()
ax_in = Axes3D(fig_in)

ax_in.set_xlim(-plot_range, plot_range)
ax_in.set_ylim(-plot_range, plot_range)
ax_in.set_zlim(-plot_range, plot_range)

if plot_mode == 0: # fixed camera = plot in CCS

    plotCamera(ax_in, np.eye(3), np.zeros((1,3)), "b", camera_size) # camera is at (0,0,0)

    for i_ex in range(len(rvec)):
        X_C = np.zeros((X_W.shape))
        for i_x in range(X_W.shape[0]):
            R_w2c = cv2.Rodrigues(rvec[i_ex])[0] # convert to the rotation matrix
            t_w2c = tvec[i_ex].reshape(3)
            X_C[i_x,:] = R_w2c.dot(X_W[i_x,:]) + t_w2c # Transform chess corners in WCS to CCS

        ax_in.plot(X_C[:,0], X_C[:,1], X_C[:,2], ".") # plot chess corners in CCS

elif plot_mode == 1: # fixed chessboard = plot in WCS

    for i_ex in range(len(rvec)):
        R_c2w = np.linalg.inv(cv2.Rodrigues(rvec[i_ex])[0]) #　世界座標系におけるカメラの向き
        t_c2w = -R_c2w.dot(tvec[i_ex]).reshape((1,3)) # 世界座標系におけるカメラの位置

        plotCamera(ax_in, R_c2w, t_c2w, "b", camera_size)

    ax_in.plot(X_W[:,0], X_W[:,1], X_W[:,2], ".")

```

    <IPython.core.display.Javascript object>

## Exercises

1. Draw the detected chess corners over the original images using `cv2.drawChessboardCorners()` for visualization.
2. Check the difference with / without the `cv2.CALIB_FIX_ASPECT_RATIO` flag.
3. Try with your own camera.
4. Try **ChArUco** patterns instead of chessboards.
   * [Detection of ChArUco Corners](https://docs.opencv.org/master/df/d4a/tutorial_charuco_detection.html)