Code for generating results for the semi-synthetic IHDP data (Figure 4(a) and Figure 4(b)).

```python
import numpy
import sympy
import pandas
import numpy as np
import pandas as pd
import sympy as sp
import datetime
import copy
import attr
import time
import logging
import itertools
import pickle
import sys
import os
import functools

import matplotlib.pyplot as plt
from matplotlib import rcParams
import collections

from scipy.optimize import minimize
from scipy.optimize import LinearConstraint
from scipy import interpolate
import warnings
from IPython.display import clear_output

HOME_DIR = ""
warnings.simplefilter(action='ignore', category=UserWarning)
```

```python
class ModelParams:

  def __init__(self, beta, a1, a2, y):
    self.beta = beta
    self.a1 = a1
    self.a2 = a2
    self.y = y

  def get_true_param(self):
    return self.beta

  def __str__(self):
    return "b=%f, a1=%f, a2=%f, y=%f" % (self.beta, self.a1, self.a2, self.y)

truth = ModelParams(beta=1, a1=1, a2=.1, y=1)
```

```python
df_ihdp = pd.read_csv(os.path.join(HOME_DIR, "data/ihdp_processed.csv"))
print(df_ihdp.shape)
df_ihdp.head()
```

    (747, 4)

      treat
      bw
      cig
      bw_std

      0
      1
      1559
      0
      -0.528957

      1
      0
      1000
      1
      -1.738109

      2
      0
      1430
      0
      -0.807992

      3
      0
      1984
      1
      0.390344

      4
      0
      1320
      1
      -1.045929

```python
def generate_data_samples(num_samples, model, k=[1/3, 1/3]):
  np = numpy
  pd = pandas

  u_y = np.random.normal(scale=np.sqrt(model.y), size=(num_samples,))
  df_idxs = np.argmax(np.random.multinomial(1, np.ones((df_ihdp.shape[0],)) / df_ihdp.shape[0],
                                            size=(num_samples,)), axis=-1)
  df = df_ihdp.iloc[df_idxs]

  def get_W1():
    return df["bw_std"].values

  def get_W2():
    return df["cig"].values

  def get_X():
    return df["treat"].values

  def get_Y(X, W1, W2, u):
    return model.beta * X + model.a1 * W1 + model.a2 * W2 + u

  W1 = get_W1()
  W2 = get_W2()
  X = get_X()
  Y = get_Y(X, W1, W2, u_y)
  SEL = np.zeros_like(Y, np.int32)

  SELS = [np.zeros_like(Y, np.int32) for _ in k]
  k_sum = 0
  for i, k_i in enumerate(k):
    prev_k_sum = k_sum
    k_sum += k_i
    SELS[i][int(prev_k_sum*Y.shape[0]) : int(k_sum*Y.shape[0])] = 1

  df = pd.DataFrame({
      "W1": W1,
      "W2": W2,
      "X": X,
      "Y": Y,
  })
  for i in range(len(k)):
    df["SEL%d" % (i+1)] = SELS[i]
  return df
```

```python
df = generate_data_samples(num_samples=10000, model=truth, k=[1/3, 1/3])
df.head()
```

      W1
      W2
      X
      Y
      SEL1
      SEL2

      0
      -1.045929
      0
      0
      -0.759767
      1
      0

      1
      -1.456911
      0
      0
      -0.313307
      1
      0

      2
      0.424953
      0
      1
      0.309768
      1
      0

      3
      -0.981037
      1
      0
      0.101697
      1
      0

      4
      -0.483533
      0
      1
      -0.076938
      1
      0

```python
def get_fraction_cost(cost=[1, 1, 1], k=[1/3, 1/3]):
  total = 0
  for c, k_val in zip(cost, k):
    total += c*k_val
  total += cost[-1]*(1 - sum(k))
  return total
```

```python
class GMMEqs:

  def __init__(self):
    self.moment_eqs, self.jacobian_eqs, self.moment_reweighting = self._get_equations()

  def get(self):
    return self.moment_eqs, self.jacobian_eqs, self.moment_reweighting

  def _get_equations(self):
    sp = sympy

    df_symbols = sp.symbols('ns1, ns2, s3, S1, S2, W1, W2, X, Y')
    ns1, ns2, s3, S1, S2, W1, W2, X, Y = df_symbols
    params = sp.symbols("b, a1, a2, t1, t2, d, w1, w2, y")
    b, a1, a2, t1, t2, d, w1, w2, y = params

    m_wts = [ns2, ns2, ns1, ns1, s3, ns2, ns1, ns2, ns1, ns2, ns1]
    moment_reweighting = sp.zeros(len(m_wts), len(m_wts))
    for i, mw1 in enumerate(m_wts):
      for j, mw2 in enumerate(m_wts):
        if mw1 == 1 or mw2 == 1:
          moment_reweighting[i, j] = mw1 * mw2
        elif mw1 == mw2:
          moment_reweighting[i, j] = mw1

    m1 = m_wts[0] * (1-S2) * (W1 * (Y - a1*W1 - b*X) - a2*d)
    m2 = m_wts[1] * (1-S2) * (X * (Y - a1*W1 - b*X) - a2*t2)
    m3 = m_wts[2] * (1-S1) * (W2 * (Y - a2*W2 - b*X) - a1*d)
    m4 = m_wts[3] * (1-S1) * (X * (Y - a2*W2 - b*X) - a1*t1)
    m5 = m_wts[4] * (1-S1-S2) * (W1*W2 - d)
    m6 = m_wts[5] * (1-S2) * (X*W1 - t1)
    m7 = m_wts[6] * (1-S1) * (X*W2 - t2)
    m8 = m_wts[7] * (1-S2) * (W1**2 - w1)
    m9 = m_wts[8] * (1-S1) * (W2**2 - w2)
    m10 = m_wts[9] * (1-S2) * ((Y - a1*W1 - b*X)**2 - a2**2 * w2 - y)
    m11 = m_wts[10] * (1-S1) * ((Y - a2*W2 - b*X)**2 - a1**2 * w1 - y)

    all_symbols = df_symbols + params
    moments = [m1, m2, m3, m4, m5, m6, m7, m8, m9, m10, m11]
    jacobian = []
    for mom in moments:
      jac_row = []
      for p in params:
        eq = sp.simplify(sp.diff(mom, p))
        jac_row.append(sp.lambdify(all_symbols, eq, "numpy"))
      jacobian.append(jac_row)

    moments = [sp.lambdify(all_symbols, eq, "numpy") for eq in moments]
    moment_reweighting = sp.lambdify((ns1, ns2, s3), moment_reweighting, "numpy")
    return moments, jacobian, moment_reweighting

class GMM:

  def __init__(self, df, gmm_eqs):
    np = numpy

    # self.df = df
    self.df_dict = {k: df[k].values for k in df.columns}
    self.df = self.df_dict
    self.moment_eqs, self.jacobian_eqs, self.moment_reweighting = gmm_eqs.get()
    self.num_samples = len(df)
    self.num_moments = len(self.moment_eqs)
    self.momconds_arr = np.zeros((self.num_moments, self.num_samples))

  def momconds(self, params):
    """Returns the emprical moment vector (equivalent of \hat{g}_n(\theta))."""
    np = numpy

    n = self.num_samples
    return np.sum(self._momconds_arr(params), axis=-1) / n

  def _compute_moment_covariance(self, params):
    """Estimate the optimal weight matrix using the emprical moments."""
    n = self.num_samples
    moms = self._momconds_arr(params)
    moment_covariance = (moms @ moms.T) / n
    return moment_covariance

  def _get_objective_fn(self, weight_matrix_inv):
    """Returns the GMM objective function."""
    np = numpy

    w_lambda = 0.001
    weight_matrix_inv += weight_matrix_inv + w_lambda * np.eye(weight_matrix_inv.shape[0])

    def objective(params):
      moms = self.momconds(params)

      w_inv_mom = np.linalg.solve(weight_matrix_inv, moms)
      obj = moms.T @ w_inv_mom

      return obj

    return objective

  def _momconds_arr(self, params):
    np = numpy

    p = params
    df = self.df_dict
    n = self.num_samples

    for i, eq in enumerate(self.moment_eqs):
      self.momconds_arr[i, :] = eq(S1=df["SEL1"], S2=df["SEL2"],
                                   W1=df["W1"], W2=df["W2"],
                                   X=df["X"], Y=df["Y"],
                                   ns1=1, ns2=1, s3=1,
                                   b=p[0], a1=p[1], a2=p[2], t1=p[3], t2=p[4], d=p[5], w1=p[6], w2=p[7], y=p[8])

    return self.momconds_arr

  def _get_asymptotic_variance_matrix(self, k, moment_covariance, params):
    np = numpy

    current_k = [np.mean(self.df["SEL1"]), np.mean(self.df["SEL2"])]

    p = params
    df = self.df
    n = self.num_samples

    jacobian = np.zeros((self.num_moments, len(p)))

    ns1_rw = (1 - k[0]) / (1 - current_k[0])
    ns2_rw = (1 - k[1]) / (1 - current_k[1])
    s3_rw = (1-k[0]-k[1])/(1-current_k[0]-current_k[1])

    for i, jac_row in enumerate(self.jacobian_eqs):
      for j, eq in enumerate(jac_row):
        jacobian[i, j] = np.sum(eq(S1=df["SEL1"], S2=df["SEL2"],
                                   W1=df["W1"], W2=df["W2"],
                                   X=df["X"], Y=df["Y"],
                                   ns1=ns1_rw, ns2=ns2_rw, s3=s3_rw,
                                   b=p[0], a1=p[1], a2=p[2], t1=p[3], t2=p[4],
                                   d=p[5], w1=p[6], w2=p[7], y=p[8])) / n

    moment_covariance_reweight = self.moment_reweighting(ns1=ns1_rw, ns2=ns2_rw, s3=s3_rw,)

    moment_covariance = moment_covariance * moment_covariance_reweight
    return np.linalg.inv(jacobian.T @ np.linalg.inv(moment_covariance) @ jacobian)

  def _get_asymptotic_variance(self, k, moment_covariance, params):
    return self._get_asymptotic_variance_matrix(k, moment_covariance, params)[0][0]

  def _optimize_find_parameters(self, weight_matrix_inv, initial_guess=None):
    np = numpy

    initial_guess = np.array([1.1, 1, 1, 1, 1, 1, 1, 1, 1]) if initial_guess is None else initial_guess
    res = minimize(self._get_objective_fn(weight_matrix_inv), initial_guess,
                   bounds=[
                           (-np.inf, np.inf),
                           (-np.inf, np.inf),
                           (-np.inf, np.inf),
                           (-np.inf, np.inf),
                           (-np.inf, np.inf),
                           (-np.inf, np.inf),
                           (0.1, np.inf),
                           (0.1, np.inf),
                           (0.1, np.inf),
                   ],
                  )
    return res.x

  def find_parameters(self, num_iters=2, weight_matrix_reg=None):
    np = numpy

    weight_matrix_inv = np.eye(self.num_moments)
    for i in range(num_iters):
      params = self._optimize_find_parameters(weight_matrix_inv, initial_guess=params if i > 0 else None)
      weight_matrix_inv = self._compute_moment_covariance(params)

      if weight_matrix_reg is not None:
        weight_matrix_inv += weight_matrix_reg * np.eye(weight_matrix_inv.shape[0])

    return params, weight_matrix_inv

  def find_optimal_k(self, moment_covariance, params, cost=[1, 1, 1]):
    np = numpy

    initial_guess = np.array([1/3, 1/3])
    lower_bound = 0.05
    upper_bound = 0.95

    if np.sum(cost) == len(cost):
      objective = lambda x: self._get_asymptotic_variance(x, moment_covariance, params)
    else:
      objective = lambda x: self._get_asymptotic_variance(x, moment_covariance, params) * (get_fraction_cost(cost, x))
    res = minimize(objective,
                   initial_guess,
                   constraints=[LinearConstraint([1, 1], [1-upper_bound], [1-lower_bound]),
                                LinearConstraint([1, 0], [lower_bound], [upper_bound]),
                                LinearConstraint([0, 1], [lower_bound], [upper_bound])],
                   )

    if res.x[0] == lower_bound and res.x[1] == lower_bound:
      return np.array([0, 0])

    return res.x
```

```python
gmm = GMM(df, GMMEqs())
params, moment_covariance = gmm.find_parameters(num_iters=2)

print("True params:")
print(truth)
print("Estimated params:")
print(params)

optimal_k = gmm.find_optimal_k(moment_covariance, params)
print("Estimated optimal selection ratio k: %s" % str(optimal_k))

del gmm, params, moment_covariance, optimal_k
```

    True params:
    b=1.000000, a1=1.000000, a2=0.100000, y=1.000000
    Estimated params:
    [ 0.96694478  1.00800977  0.19191733  0.04045318  0.07599427 -0.06614089
      1.01743196  0.36492049  0.99095177]
    Estimated optimal selection ratio k: [0.05 0.05]

```python
class SampleRevealer:

  def __init__(self, budget, cost, df):
    self.initial_budget = budget
    self.budget = budget
    self.cost = cost
    self.buffer_size = len(df)
    self.df = df
    self.counter = 0

  def reset(self):
    self.counter = 0
    self.budget = self.initial_budget

  def is_budget_left(self, samples_to_reveal, k):
    return self.budget >= samples_to_reveal * get_fraction_cost(self.cost, k)

  def reveal(self, reveal_k, samples_to_reveal):
    if self.counter + samples_to_reveal >= self.buffer_size:
      raise ValueError("no buffer")

    if not self.is_budget_left(samples_to_reveal, reveal_k):
      raise ValueError("no budget left")

    observe_count_1 = int(reveal_k[0] * samples_to_reveal)
    observe_count_2 = int(reveal_k[1] * samples_to_reveal)
    self.df["SEL1"][self.counter:self.counter+samples_to_reveal].values[:] = 0
    self.df["SEL2"][self.counter:self.counter+samples_to_reveal].values[:] = 0
    self.df["SEL1"].values[self.counter:self.counter+observe_count_1] = 1
    self.df["SEL2"].values[self.counter+observe_count_1:self.counter+observe_count_1+observe_count_2] = 1

    self.counter += samples_to_reveal
    self.budget -= (samples_to_reveal * get_fraction_cost(self.cost, reveal_k))

  def reveal_with_budget(self, reveal_k, budget_to_reveal):
    samples_to_reveal = int(budget_to_reveal / get_fraction_cost(self.cost, reveal_k))
    self.reveal(reveal_k, samples_to_reveal)

  def get_dataset(self):
    return self.df[:self.counter]
```

```python
def batch_fractions_to_sizes(horizon, batch_fractions):
  batch_sizes = []

  for i in range(len(batch_fractions) + 1):
    prev_batch_end = 0 if i == 0 else int(horizon * batch_fractions[i - 1])
    curr_batch_end = horizon if i == len(batch_fractions) else int(horizon * batch_fractions[i])

    batch_sizes.append(curr_batch_end - prev_batch_end)

  return batch_sizes
```

```python
def k_to_simplex(k):
  np = numpy

  k_simplex = np.zeros((len(k) + 1))
  k_simplex[:-1] = k
  k_simplex[-1] = 1 - np.sum(k)
  return k_simplex

def compute_reveal_k(current_samples, samples_to_reveal, current_k, target_k):
  np = numpy

  current_k = np.array(current_k)
  target_k = np.array(target_k)

  target_simplex = k_to_simplex(target_k)

  def objective(k):
    k_final = (current_k * current_samples + k * samples_to_reveal) / (current_samples + samples_to_reveal)
    diff = k_to_simplex(k_final) - target_simplex
    return np.dot(diff, diff)

  lower_bound = 0.01
  upper_bound = 0.99
  res = minimize(objective,
                 current_k,
                 constraints=[LinearConstraint([1, 1], [0], [1]),
                              LinearConstraint([1, 0], [lower_bound], [upper_bound]),
                              LinearConstraint([0, 1], [lower_bound], [upper_bound])],
                 )

  return res.x

def compute_reveal_k_with_budget(current_samples, budget_to_reveal, cost, current_k, target_k):
  np = numpy

  np = numpy

  current_k = np.array(current_k)
  target_k = np.array(target_k)

  target_simplex = k_to_simplex(target_k)

  def objective(k):
    samples_to_reveal = int(budget_to_reveal / get_fraction_cost(cost, k))
    k_final = (current_k * current_samples + k * samples_to_reveal) / (current_samples + samples_to_reveal)
    diff = k_to_simplex(k_final) - target_simplex
    return np.dot(diff, diff)

  lower_bound = 0.01
  upper_bound = 0.99
  res = minimize(objective,
                 current_k,
                 constraints=[LinearConstraint([1, 1], [0], [1]),
                              LinearConstraint([1, 0], [lower_bound], [upper_bound]),
                              LinearConstraint([0, 1], [lower_bound], [upper_bound])],
                 )

  return res.x
```

```python
class StrategyRunResult:

  def __init__(self):
    self.budgets_used = []
    self.squared_errors = []
    self.optimal_ks = []
    self.current_ks = []

  def append(self, budget_used, squared_error, optimal_k, current_k):
    self.budgets_used.append(budget_used)
    self.squared_errors.append(squared_error)
    self.optimal_ks.append(optimal_k)
    self.current_ks.append(current_k)
```

```python
class OracleStrategy:

  def __init__(self, sample_revealer, gmm_equations, optimal_k, cost, budget, batch_fractions):
    self.sample_revealer = sample_revealer
    self.name = 'oracle'
    self.gmm_equations = gmm_equations

    # Batches are budget fraction allocations.
    self.batch_sizes = batch_fractions_to_sizes(budget, batch_fractions)
    self.optimal_k = optimal_k
    self.cost = cost

  def get_current_df_vals(self):
    np = numpy

    dataset = self.sample_revealer.get_dataset()

    sel1 = np.sum(dataset["SEL1"])
    sel2 = np.sum(dataset["SEL2"])
    total = len(dataset)
    return np.array([sel1/total, sel2/total]), total

  def can_step(self):
    return self.sample_revealer.is_budget_left()

  def get_and_store_params(self):
    dataset = self.sample_revealer.get_dataset()
    self.gmm = GMM(dataset, self.gmm_equations)
    self.params, self.moment_covariance = self.gmm.find_parameters(num_iters=2)
    return self.params

  def get_squared_error(self):
    error = (truth.get_true_param() - self.params[0])**2
    return error

  def execute_run(self, cost=1):
    np = numpy

    result = StrategyRunResult()
    for i, batch_size in enumerate(self.batch_sizes):
      if i == 0:
        current_k, current_samples = np.array([0, 0]), 0
      else:
        current_k, current_samples = self.get_current_df_vals()

      reveal_k = compute_reveal_k_with_budget(current_samples, batch_size, self.cost, current_k, self.optimal_k)

      self.sample_revealer.reveal_with_budget(reveal_k=reveal_k, budget_to_reveal=batch_size)
      _ = self.get_and_store_params()

      current_k, _ = self.get_current_df_vals()

      result.append(
          self.sample_revealer.initial_budget - self.sample_revealer.budget,
          self.get_squared_error(), self.optimal_k, current_k,
      )

    return result
```

```python
class ETCStrategy:

  def __init__(self, sample_revealer, gmm_equations, cost, budget, batch_fractions):
    self.sample_revealer = sample_revealer
    self.name = 'etc'
    self.gmm_equations = gmm_equations
    # Batch sizes represent fractions of budget allocations.
    # We assume that the first batch size is exploration.
    self.batch_sizes = batch_fractions_to_sizes(budget, batch_fractions)
    self.cost = cost

  def get_current_df_vals(self):
    np = numpy

    dataset = self.sample_revealer.get_dataset()

    sel1 = np.sum(dataset["SEL1"])
    sel2 = np.sum(dataset["SEL2"])
    total = len(dataset)
    return np.array([sel1/total, sel2/total]), total

  def can_step(self):
    return self.sample_revealer.is_budget_left()

  def get_and_store_params(self):
    dataset = self.sample_revealer.get_dataset()
    self.gmm = GMM(dataset, self.gmm_equations)
    self.params, self.moment_covariance = self.gmm.find_parameters(num_iters=2)
    return self.params

  def get_squared_error(self):
    error = (truth.get_true_param() - self.params[0])**2
    return error

  def execute_run(self, cost=1):
    np = numpy

    result = StrategyRunResult()
    for i, batch_size in enumerate(self.batch_sizes):
      if i == 0:
        reveal_k = [0, 0]
      else:
        current_k, current_samples = self.get_current_df_vals()
        reveal_k = compute_reveal_k_with_budget(current_samples, batch_size,
                                                self.cost, current_k, self.optimal_k)

      self.sample_revealer.reveal_with_budget(reveal_k=reveal_k, budget_to_reveal=batch_size)
      params = self.get_and_store_params()

      if i == 0:
        df = self.sample_revealer.get_dataset()
        if np.mean(df["X"]) <= 0.1:
          return None

        self.optimal_k = self.gmm.find_optimal_k(self.moment_covariance, params, cost=cost)

      current_k, _ = self.get_current_df_vals()

      result.append(
          self.sample_revealer.initial_budget - self.sample_revealer.budget,
          self.get_squared_error(), self.optimal_k, current_k,
      )

    return result
```

```python
class ETGreedyFBStrategy:

  def __init__(self, sample_revealer, gmm_equations, cost, budget, batch_fractions):
    self.sample_revealer = sample_revealer
    self.name = 'etg-fb'
    self.gmm_equations = gmm_equations
    # We assume that the first batch size is exploration.
    self.batch_sizes = batch_fractions_to_sizes(budget, batch_fractions)
    self.cost = cost
    self.weight_matrix_reg = 0.01

  def get_current_df_vals(self):
    np = numpy

    dataset = self.sample_revealer.get_dataset()

    sel1 = np.sum(dataset["SEL1"])
    sel2 = np.sum(dataset["SEL2"])
    total = len(dataset)
    return np.array([sel1/total, sel2/total]), total

  def can_step(self):
    return self.sample_revealer.is_budget_left()

  def get_and_store_params(self, is_last_step):
    dataset = self.sample_revealer.get_dataset()
    self.gmm = GMM(dataset, self.gmm_equations)
    if is_last_step:
      self.params, self.moment_covariance = self.gmm.find_parameters(num_iters=2)
    else:
      self.params, self.moment_covariance = (
          self.gmm.find_parameters(num_iters=2, weight_matrix_reg=self.weight_matrix_reg)
      )
    return self.params

  def get_squared_error(self):
    error = (truth.get_true_param() - self.params[0])**2
    return error

  def execute_run(self, cost=1):
    np = numpy

    result = StrategyRunResult()
    for i, batch_size in enumerate(self.batch_sizes):
      if i == 0:
        reveal_k = [0, 0]
      else:
        current_k, current_samples = self.get_current_df_vals()
        reveal_k = compute_reveal_k_with_budget(current_samples, batch_size, self.cost, current_k, self.optimal_k)

      self.sample_revealer.reveal_with_budget(reveal_k=reveal_k, budget_to_reveal=batch_size)
      if i == 0:
        df = self.sample_revealer.get_dataset()
        if np.mean(df["X"]) <= 0.1:
          return None

      is_last_step = (i==len(self.batch_sizes)-1)
      params = self.get_and_store_params(is_last_step)

      self.optimal_k = self.gmm.find_optimal_k(self.moment_covariance, params, cost=cost)

      current_k, _ = self.get_current_df_vals()

      result.append(
          self.sample_revealer.initial_budget - self.sample_revealer.budget,
          self.get_squared_error(), self.optimal_k, current_k,
      )

    return result
```

```python
class ETGreedyFSStrategy:

  def __init__(self, sample_revealer, gmm_equations, cost, budget, batch_fractions):
    np = numpy

    self.sample_revealer = sample_revealer
    self.name = 'etg-fs'
    horizon_min = int(budget / np.max(cost))
    # We assume that the first batch size is exploration.
    self.batch_sizes = batch_fractions_to_sizes(horizon_min, batch_fractions)[:-1]
    self.cost = cost
    self.gmm_equations = gmm_equations
    self.weight_matrix_reg = 0.01

  def get_current_df_vals(self):
    np = numpy

    dataset = self.sample_revealer.get_dataset()

    sel1 = np.sum(dataset["SEL1"])
    sel2 = np.sum(dataset["SEL2"])
    total = len(dataset)
    return np.array([sel1/total, sel2/total]), total

  def can_step(self):
    return self.sample_revealer.is_budget_left()

  def get_and_store_params(self, is_last_step):
    dataset = self.sample_revealer.get_dataset()
    self.gmm = GMM(dataset, self.gmm_equations)
    if is_last_step:
      self.params, self.moment_covariance = self.gmm.find_parameters(num_iters=2)
    else:
      self.params, self.moment_covariance = (
          self.gmm.find_parameters(num_iters=2, weight_matrix_reg=self.weight_matrix_reg)
      )
    return self.params

  def get_squared_error(self):
    error = (truth.get_true_param() - self.params[0])**2
    return error

  def execute_run(self, cost=1):
    np = numpy

    result = StrategyRunResult()
    for i, batch_size in enumerate(self.batch_sizes):
      if i == 0:
        reveal_k = [0, 0]
      else:
        current_k, current_samples = self.get_current_df_vals()
        reveal_k = compute_reveal_k(current_samples, batch_size, current_k, self.optimal_k)

      self.sample_revealer.reveal(reveal_k=reveal_k, samples_to_reveal=batch_size)
      if i == 0:
        df = self.sample_revealer.get_dataset()
        if np.mean(df["X"]) <= 0.1:
          return None

      params = self.get_and_store_params(is_last_step=False)

      self.optimal_k = self.gmm.find_optimal_k(self.moment_covariance, params, cost=cost)

      current_k, current_samples = self.get_current_df_vals()

      result.append(
          self.sample_revealer.initial_budget - self.sample_revealer.budget,
          self.get_squared_error(), self.optimal_k, current_k,
      )

    last_step_reached = False
    batch_size = self.batch_sizes[-1]
    while not last_step_reached:
      current_k, current_samples = self.get_current_df_vals()
      reveal_k = compute_reveal_k(current_samples, batch_size, current_k, self.optimal_k)

      if self.sample_revealer.is_budget_left(samples_to_reveal=batch_size, k=reveal_k):
        self.sample_revealer.reveal(reveal_k=reveal_k, samples_to_reveal=batch_size)
      else:
        budget_left = self.sample_revealer.budget
        reveal_k = compute_reveal_k_with_budget(current_samples, budget_left, self.cost, current_k, self.optimal_k)
        self.sample_revealer.reveal_with_budget(reveal_k=reveal_k, budget_to_reveal=budget_left)
        last_step_reached = True

      params = self.get_and_store_params(last_step_reached)
      self.optimal_k = self.gmm.find_optimal_k(self.moment_covariance, params, cost=cost)

      current_k, current_samples = self.get_current_df_vals()

      result.append(
          self.sample_revealer.initial_budget - self.sample_revealer.budget,
          self.get_squared_error(), self.optimal_k, current_k,
      )

    return result
```

```python
def execute_strategy_iteration(strategy_name, iteration_num, budget):
  np = numpy

  cost = [1, 3, 3.5]

  gmm_equations = GMMEqs()

  # Uncomment the next two lines to replicate the exact runs used in the paper.
  # random_seed = 232281293 + iteration_num
  # np.random.seed(random_seed)
  df = generate_data_samples(num_samples=budget * 5, model=truth)
  np.random.seed(None)

  sample_revealer = SampleRevealer(budget=budget, cost=cost, df=df)

  if strategy_name == "oracle":
    strategy = OracleStrategy(sample_revealer, gmm_equations=gmm_equations,
                              budget=budget, cost=cost, optimal_k=[0.59408789, 0],
                              batch_fractions=[0.1, 0.9])
  elif strategy_name == "collect_all":
    strategy = OracleStrategy(sample_revealer, gmm_equations=gmm_equations,
                              budget=budget, cost=cost, optimal_k=[0, 0],
                              batch_fractions=[0.1, 0.2, 0.5, 0.9])
  elif strategy_name == "etc_0.2":
    strategy = ETCStrategy(sample_revealer, gmm_equations=gmm_equations,
                           budget=budget, cost=cost,
                           batch_fractions=[0.2, 0.9])
  elif strategy_name == "etc_0.4":
    strategy = ETCStrategy(sample_revealer, gmm_equations=gmm_equations,
                           budget=budget, cost=cost,
                           batch_fractions=[0.4, 0.9])
  elif strategy_name == "etg-fb_0.4":
    strategy = ETGreedyFBStrategy(sample_revealer, gmm_equations=gmm_equations,
                                  budget=budget, cost=cost,
                                  batch_fractions=[0.4, 0.5, 0.6, 0.7, 0.8])
  elif strategy_name == "etg-fs_0.4":
    strategy = ETGreedyFSStrategy(sample_revealer, gmm_equations=gmm_equations,
                                  budget=budget, cost=cost,
                                  batch_fractions=[0.4, 0.8])
  else:
    raise ValueError("invalid strategy_name: %s" % strategy_name)

  return strategy.execute_run(cost=cost)
```

```python
# Test a strategy.

result = execute_strategy_iteration("etc_0.4", 1, budget=1000)
print(result.squared_errors)
```

    [0.00555592535273502, 0.002185244501872844, 0.0024872469563182687]

**Executing the runs for each strategy in parallel.**

For the paper, we execute 12,000 runs for each strategy. We execute the runs
in parallel using
[`ipyparallel`](https://ipyparallel.readthedocs.io/en/latest/).

The following command starts the `ipyparallel` engines:
```
ipcluster start -n <num_engines>
```

```python
import ipyparallel as ipp
```

```python
# Verify that ipcluster is running and import the necessary Python packages.

parallel_client = ipp.Client(debug=False)
dview = parallel_client[:]
# Execute an identity map in parallel.
ar = dview.map(lambda x: x, (i for i in range(0, 2000000, 2)))
assert ar.get()[0] == 0

# Import the required Python packages.
with dview.sync_imports():
  from abc import ABC, abstractmethod
  import numpy
  import sympy
  import pandas
  import sympy
  import datetime
  import copy
  import attr
  import time
  import logging
  import itertools
  import pickle
  import os
  import functools
  import ipyparallel

  import collections

  from scipy.optimize import minimize
  from scipy import interpolate
  from scipy.special import expit
  from scipy.optimize import LinearConstraint
  import warnings

  try:
    from cPickle import dumps, loads, HIGHEST_PROTOCOL as PICKLE_PROTOCOL
  except ImportError:
    from pickle import dumps, loads, HIGHEST_PROTOCOL as PICKLE_PROTOCOL

# Make sure ipyparallel is still able to execute functions.
dview = parallel_client[:]
ar = dview.map(lambda x: x, (i for i in range(0, 2000000, 2)))
assert ar.get()[0] == 0
```

    importing ABC,abstractmethod from abc on engine(s)
    importing numpy on engine(s)
    importing sympy on engine(s)
    importing pandas on engine(s)
    importing datetime on engine(s)
    importing copy on engine(s)
    importing attr on engine(s)
    importing time on engine(s)
    importing logging on engine(s)
    importing itertools on engine(s)
    importing pickle on engine(s)
    importing os on engine(s)
    importing functools on engine(s)
    importing ipyparallel on engine(s)
    importing collections on engine(s)
    importing minimize from scipy.optimize on engine(s)
    importing interpolate from scipy on engine(s)
    importing expit from scipy.special on engine(s)
    importing LinearConstraint from scipy.optimize on engine(s)
    importing warnings on engine(s)

```python
def combine_parallel_results(async_result, need_interpolation=False):
  optimal_ks_res = []

  if not need_interpolation:
    budgets = async_result.get()[0].budgets_used
    errors = np.vstack([res.squared_errors for res in async_result.get() if res is not None])
    optimal_ks = np.vstack([[res.optimal_ks] for res in async_result.get() if res is not None])
    current_ks = np.vstack([[res.current_ks] for res in async_result.get() if res is not None])

    return budgets, errors, optimal_ks, current_ks

  budgets_matrix = [res.budgets_used for res in async_result.get() if res is not None]
  errors_matrix = [res.squared_errors for res in async_result.get() if res is not None]
  optimal_ks_matrix = [res.optimal_ks for res in async_result.get() if res is not None]
  current_ks_matrix = [res.current_ks for res in async_result.get() if res is not None]

  def find_budget():
    idx = 0
    target = budgets_matrix[0][-1]

    for i in range(1, len(budgets_matrix)):
      if target > budgets_matrix[i][-1]:
        target = budgets_matrix[i][-1]
        idx = i

    return np.array(budgets_matrix[idx])

  budgets = find_budget()
  errors = []
  optimal_ks = []
  current_ks = []

  for i in range(len(budgets_matrix)):
    f_err = interpolate.interp1d(budgets_matrix[i], errors_matrix[i])
    f_optimalk0 = interpolate.interp1d(budgets_matrix[i], np.array(optimal_ks_matrix[i])[:, 0])
    f_optimalk1 = interpolate.interp1d(budgets_matrix[i], np.array(optimal_ks_matrix[i])[:, 1])
    f_currentk0 = interpolate.interp1d(budgets_matrix[i], np.array(current_ks_matrix[i])[:, 0])
    f_currentk1 = interpolate.interp1d(budgets_matrix[i], np.array(current_ks_matrix[i])[:, 1])

    errors.append([])
    optimal_ks.append([])
    current_ks.append([])

    for b in budgets:
      errors[i].append(f_err(b))
      optimal_ks[i].append([f_optimalk0(b), f_optimalk1(b)])
      current_ks[i].append([f_currentk0(b), f_currentk0(b)])

  errors = np.array(errors)
  optimal_ks = np.array(optimal_ks)
  current_ks = np.array(current_ks)

  return budgets, errors, optimal_ks, current_ks

def execute_strategy_in_parallel(strategy_name, horizon, iterations):
  num_threads = len(parallel_client.ids)
  dview = parallel_client[:]

  print("Executing %s over %d iterations across %d cores" % (strategy_name, iterations, num_threads))

  dview["batch_fractions_to_sizes"] = batch_fractions_to_sizes
  dview["compute_reveal_k"] = compute_reveal_k
  dview["compute_reveal_k_with_budget"] = compute_reveal_k_with_budget
  dview["get_fraction_cost"] = get_fraction_cost
  dview["k_to_simplex"] = k_to_simplex
  dview["ModelParams"] = ModelParams
  dview["truth"] = truth
  dview["df_ihdp"] = df_ihdp
  dview["generate_data_samples"] = generate_data_samples
  dview["StrategyRunResult"] = StrategyRunResult
  dview["GMM"] = GMM
  dview["GMMEqs"] = GMMEqs
  dview["OracleStrategy"] = OracleStrategy
  dview["ETCStrategy"] = ETCStrategy
  dview["ETGreedyFBStrategy"] = ETGreedyFBStrategy
  dview["ETGreedyFSStrategy"] = ETGreedyFSStrategy
  dview["SampleRevealer"] = SampleRevealer
  dview["execute_strategy_iteration"] = execute_strategy_iteration

  def execute_iteration(i):
    return execute_strategy_iteration(strategy_name, i, horizon)

  return dview.map(execute_iteration, range(iterations))
```

```python
def get_timeseries_for(strategy_names, horizons, iterations, results_dict):
  results_dict["truth"] = truth

  for strategy_name in strategy_names:
    for horizon in horizons:
      print("Timestamp start: %s, Strategy: %s, Horizon: %d, Iters: %d" % (
          datetime.datetime.now(), strategy_name, horizon, iterations))
      async_result = execute_strategy_in_parallel(strategy_name, horizon, iterations)
      need_interpolation = "etg-fs" in strategy_name
      budgets, errors, optimal_ks, current_ks = combine_parallel_results(async_result, need_interpolation)
      print("Timestamp end: %s" % (datetime.datetime.now()))

      if strategy_name not in results_dict:
        results_dict[strategy_name] = {}

      results_dict[strategy_name][horizon] = {
          "budgets": budgets,
          "errors": errors * budgets,
          "optimal_ks": optimal_ks,
          "current_ks": current_ks,
      }
```

```python
results_dict = {}
get_timeseries_for([
                    "oracle", "collect_all",
                    "etc_0.2", "etc_0.4",
                    "etg-fb_0.4", "etg-fs_0.4",
                    ],
                   horizons=[500, 600, 700, 800, 900, 1000, 1150, 1300],
                   iterations=12*1000,
                   results_dict=results_dict)
```

```python
# Save results to file.
# pickle.dump(results_dict,
#             open(os.path.join(HOME_DIR,
#                               "%s_ihdp_graph.pkl" % datetime.datetime.now()),
#                  "wb"))
```

### Plot the results

```python
# For the color map:
# https://gist.github.com/AndiH/c957b4d769e628f506bd

# Tableau 20 Colors
tableau20 = [(31, 119, 180), (174, 199, 232), (255, 127, 14), (255, 187, 120),
             (44, 160, 44), (152, 223, 138), (214, 39, 40), (255, 152, 150),
             (148, 103, 189), (197, 176, 213), (140, 86, 75), (196, 156, 148),
             (227, 119, 194), (247, 182, 210), (127, 127, 127), (199, 199, 199),
             (188, 189, 34), (219, 219, 141), (23, 190, 207), (158, 218, 229)]

# Tableau Color Blind 10
tableau20blind = [(0, 107, 164), (255, 128, 14), (171, 171, 171), (89, 89, 89),
             (95, 158, 209), (200, 82, 0), (137, 137, 137), (163, 200, 236),
             (255, 188, 121), (207, 207, 207)]

# Rescale to values between 0 and 1
for i in range(len(tableau20)):
    r, g, b = tableau20[i]
    tableau20[i] = (r / 255., g / 255., b / 255.)
for i in range(len(tableau20blind)):
    r, g, b = tableau20blind[i]
    tableau20blind[i] = (r / 255., g / 255., b / 255.)
# Use with plt.plot(…, color=tableau[0],…)
```

```python
def plot_regret_curve(results_dict):

  clist = rcParams['axes.prop_cycle']
  cgen = itertools.cycle(clist)

  oracle_mses = {}
  for horizon, timeseries in results_dict["oracle"].items():
    oracle_mses[timeseries["budgets"][-1]] = np.mean(timeseries["errors"][:, -1])

  oracle_budgets = np.array(list(oracle_mses.keys()))

  def plot(axs, name, info, result, with_var=False):
    x_vals = []
    y_mean = []
    y_std = []

    current_k_mean = []
    optimal_k_mean = []

    for horizon, timeseries in result.items():
      x_vals.append(timeseries["budgets"][-1])

      closest_budget = oracle_budgets[np.argmin(np.abs(oracle_budgets - x_vals[-1]))]
      y_scaled = ( timeseries["errors"][:, -1] - oracle_mses[closest_budget] )  / oracle_mses[closest_budget] * 100
      y_mean.append(np.mean(y_scaled))
      y_std.append(np.std(y_scaled))

    x_vals = np.array(x_vals)
    y_mean = np.array(y_mean)
    y_std = np.array(y_std)

    x_sort_idx = np.argsort(x_vals)
    x_vals = x_vals[x_sort_idx]
    y_mean = y_mean[x_sort_idx]
    y_std = y_std[x_sort_idx]

    color = next(cgen)["color"]

    plt.plot(x_vals, y_mean, label=name, color=info[1], linestyle=info[0],  marker=info[2])
    plt.legend()

    if with_var:
      ci = 1.96 * y_std / np.sqrt(timeseries["errors"].shape[0])
      axs.errorbar(x_vals, y_mean, yerr=ci, ls="none", color=info[1])

  name_to_linestyle_color = {
      "etc_0.2": ["dashdot", tableau20blind[1], "s"],
      "etc_0.4": ["solid", tableau20blind[2], "v"],
      "etg-fb_0.4": ["solid", tableau20blind[8], "^"],
      "etg-fs_0.4": ["dashdot", tableau20blind[6], "D"],
      "collect_all": ["dashed", tableau20blind[7], "o"],
  }
  plt.title("Relative regret vs budget")
  plt.xlabel("Total budget ")
  plt.ylabel("Relative regret (%)")
  for name, info in name_to_linestyle_color.items():
    plot(plt, name, info, results_dict[name], with_var=True)

  # plt.savefig(os.path.join(HOME_DIR, "figures/ihdp_regret_curve.eps"), bbox_inches='tight', pad_inches=0.0)
```

```python
SMALL_SIZE = 12
MEDIUM_SIZE = 12
BIGGER_SIZE = 12

plt.rc('font', size=SMALL_SIZE)          # controls default text sizes
plt.rc('axes', titlesize=SMALL_SIZE+4)     # fontsize of the axes title
plt.rc('axes', labelsize=MEDIUM_SIZE+4)    # fontsize of the x and y labels
plt.rc('xtick', labelsize=SMALL_SIZE)    # fontsize of the tick labels
plt.rc('ytick', labelsize=SMALL_SIZE)    # fontsize of the tick labels
plt.rc('legend', fontsize=SMALL_SIZE)    # legend fontsize
plt.rc('figure', titlesize=BIGGER_SIZE+20)  # fontsize of the figure title
```

```python
plot_regret_curve(results_dict)
```


**You may need to install [OpenCV](https://pypi.python.org/pypi/opencv-python) and [scikit-video](http://www.scikit-video.org/stable/).**

```python
import keras
import numpy as np
import io
import base64
from IPython.display import HTML
import skvideo.io
import cv2
import json

from keras.models import Sequential, model_from_json
from keras.layers.core import Dense
from keras.optimizers import sgd
from keras.layers import Conv2D, MaxPooling2D, Activation, AveragePooling2D, \
                         Reshape, BatchNormalization, Flatten
import keras.backend as K
import tensorflow as tf
```

    Using TensorFlow backend.

# MiniProject #3: Deep Reinforcement Learning

__Notations__: $E_p$ is the expectation under probability $p$. Please justify each of your answer and widely comment your code.

# Context

In a reinforcement learning algorithm, we modelize each step $t$ as an action $a_t$ obtained from a state $s_t$, i.e. $\{(a_{t},s_{t})_{t\leq T}\}$ having the Markov property. We consider a discount factor $\gamma \in [0,1]$ that ensures convergence. The goal is to find among all the policies $\pi$, one that maximizes the expected reward:

\begin{equation*}
R(\pi)=\sum_{t\leq T}E_{p^{\pi}}[\gamma^t r(s_{t},a_{t})] \> ,
\end{equation*}

where:
\begin{equation*}p^{\pi}(a_{0},a_{1},s_{1},...,a_{T},s_{T})=p(a_{0})\prod_{t=1}^{T}\pi(a_{t}|s_{t})p(s_{t+1}|s_{t},a_{t}) \> .
\end{equation*}

We note the $Q$-function:

\begin{equation*}Q^\pi(s,a)=E_{p^{\pi}}[\sum_{t\leq T}\gamma^{t}r(s_{t},a_{t})|s_{0}=s,a_{0}=a] \> .
\end{equation*}

Thus, the optimal Q function is:
\begin{equation*}
Q^*(s,a)=\max_{\pi}Q^\pi(s,a) \> .
\end{equation*}

In this project, we will apply the deep reinforcement learning techniques to a simple game: an agent will have to learn from scratch a policy that will permit it maximizing a reward.

## The environment, the agent and the game

### The environment

```Environment``` is an abstract class that represents the states, rewards, and actions to obtain the new state.

```python
class Environment(object):
    def __init__(self):
        pass

    def act(self, act):
        """
        One can act on the environment and obtain its reaction:
        - the new state
        - the reward of the new state
        - should we continue the game?

        :return: state, reward, game_over
        """
        pass

    def reset(self):
        """
        Reinitialize the environment to a random state and returns
        the original state

        :return: state
        """
        pass

    def draw(self):
        """
        Visualize in the console or graphically the current state
        """
        pass
```

The method ```act``` allows to act on the environment at a given state $s_t$ (stored internally), via action $a_t$. The method will return the new state $s_{t+1}$, the reward $r(s_{t},a_{t})$ and determines if $t\leq T$ (*game_over*).

The method ```reset``` simply reinitializes the environment to a random state $s_0$.

The method ```draw``` displays the current state $s_t$ (this is useful to check the behavior of the Agent).

We modelize $s_t$ as a tensor, while $a_t$ is an integer.

### The Agent

The goal of the ```Agent``` is to interact with the ```Environment``` by proposing actions $a_t$ obtained from a given state $s_t$ to attempt to maximize its __reward__ $r(s_t,a_t)$. We propose the following abstract class:

```python
class Agent(object):
    def __init__(self, epsilon=0.1, n_action=4):
        self.epsilon = epsilon
        self.n_action = n_action

    def set_epsilon(self, e):
        self.epsilon = e

    def act(self, s, train=True):
        """ This function should return the next action to do:
        an integer between 0 and 4 (not included) with a random exploration of epsilon"""
        if train:
            if np.random.rand() <= self.epsilon:
                a = np.random.randint(0, self.n_action, size=1)[0]
            else:
                a = self.learned_act(s)
        else: # in some cases, this can improve the performance.. remove it if poor performances
            a = self.learned_act(s)

        return a

    def learned_act(self,s):
        """ Act via the policy of the agent, from a given state s
        it proposes an action a"""
        pass

    def reinforce(self, s, n_s, a, r, game_over_):
        """ This function is the core of the learning algorithm.
        It takes as an input the current state s_, the next state n_s_
        the action a_ used to move from s_ to n_s_ and the reward r_.

        Its goal is to learn a policy.
        """
        pass

    def save(self):
        """ This function returns basic stats if applicable: the
        loss and/or the model"""
        pass

    def load(self):
        """ This function allows to restore a model"""
        pass
```

***
__Question 1__:
Explain the function act. Why is ```epsilon``` essential?

The function ***act*** represents basically the policy. Indeed, according to the current state s, act returns the action the ```Agent``` must use. Here, the function ***act*** is divided in two steps. During the training step, the function ***act*** is a trade-off between exploration and exploitation. The goal of epsilon is precisely to explore more state: the ```Agent``` is going to use a random action with probability epsilon. Otherwise, the ```Agent``` uses the learned policy (***learned_act***). Therefore, the goal of epsilon is to encourage the exploration during the training period.

***
### The Game

The ```Agent``` and the ```Environment``` work in an interlaced way as in the following (take some time to understand this code as it is the core of the project)

```python

epoch = 300
env = Environment()
agent = Agent()

# Number of won games
score = 0
loss = 0

for e in range(epoch):
    # At each epoch, we restart to a fresh game and get the initial state
    state = env.reset()
    # This assumes that the games will end
    game_over = False

    win = 0
    lose = 0

    while not game_over:
        # The agent performs an action
        action = agent.act(state)

        # Apply an action to the environment, get the next state, the reward
        # and if the games end
        prev_state = state
        state, reward, game_over = env.act(action)

        # Update the counters
        if reward > 0:
            win = win + reward
        if reward < 0:
            lose = lose -reward

        # Apply the reinforcement strategy
        loss = agent.reinforce(prev_state, state,  action, reward, game_over)

    # Save as a mp4
    if e % 10 == 0:
        env.draw(e)

    # Update stats
    score += win-lose

    print("Epoch {:03d}/{:03d} | Loss {:.4f} | Win/lose count {}/{} ({})"
          .format(e, epoch, loss, win, lose, win-lose))
    agent.save()
```

# The game, *eat cheese*

A rat runs on an island and tries to eat as much as possible. The island is subdivided into $N\times N$ cells, in which there are cheese (+0.5) and poisonous cells (-1). The rat has a visibility of 2 cells (thus it can see $5^2$ cells). The rat is given a time $T$ to accumulate as much food as possible. It can perform 4 actions: going up, down, left, right.

The goal is to code an agent to solve this task that will learn by trial and error. We propose the following environment:

```python
class Environment(object):
    def __init__(self, grid_size=10, max_time=500, temperature=0.1):
        grid_size = grid_size + 4
        self.grid_size = grid_size
        self.max_time = max_time
        self.temperature = temperature

        #board on which one plays
        self.board = np.zeros((grid_size, grid_size))
        self.position = np.zeros((grid_size, grid_size))

        # coordinate of the cat
        self.x = 0
        self.y = 1

        # self time
        self.t = 0

        self.scale = 16

        self.to_draw = np.zeros((max_time + 2, grid_size * self.scale,
                                 grid_size * self.scale, 3))

    def draw(self, e):
        skvideo.io.vwrite(str(e) + '.mp4', self.to_draw)

    def get_frame(self,t):
        b = np.zeros((self.grid_size, self.grid_size, 3)) + 128
        b[self.board > 0, 0] = 256
        b[self.board < 0, 2] = 256
        b[self.x, self.y, :] = 256
        b[-2:, :, :] = 0
        b[:, -2:, :] = 0
        b[:2, :, :] = 0
        b[:, :2, :] = 0

        b = cv2.resize(b, None, fx=self.scale, fy=self.scale,
                       interpolation=cv2.INTER_NEAREST)

        self.to_draw[t, :, :, :] = b

    def act(self, action):
        """This function returns the new state, reward and decides if the
        game ends."""

        self.get_frame(int(self.t))

        self.position = np.zeros((self.grid_size, self.grid_size))

        self.position[0:2, :]= -1
        self.position[:, 0:2] = -1
        self.position[-2:, :] = -1
        self.position[-2:, :] = -1

        self.position[self.x, self.y] = 1

        if action == 0: # Going up (or down on the image)
            if self.x == self.grid_size - 3:
                self.x = self.x - 1
            else:
                self.x = self.x + 1
        elif action == 1: # Going up on the image
            if self.x == 2:
                self.x = self.x + 1
            else:
                self.x = self.x - 1
        elif action == 2: # Going right on the image
            if self.y == self.grid_size - 3:
                self.y = self.y - 1
            else:
                self.y = self.y + 1
        elif action == 3: # Going left on the image
            if self.y == 2:
                self.y = self.y + 1
            else:
                self.y = self.y - 1
        else:
            RuntimeError('Error: action not recognized')

        self.t = self.t + 1
        reward = self.board[self.x, self.y]
        self.board[self.x, self.y] = 0
        game_over = self.t > self.max_time
        state = np.concatenate((self.board.reshape(self.grid_size, self.grid_size, 1),
                                self.position.reshape(self.grid_size, self.grid_size, 1)),
                               axis=2)
        state = state[self.x-2:self.x+3, self.y-2:self.y+3, :]

        return state, reward, game_over

    def reset(self):
        """This function resets the game and returns the initial state"""

        self.x = np.random.randint(3, self.grid_size-3, size=1)[0]
        self.y = np.random.randint(3, self.grid_size-3, size=1)[0]

        bonus = 0.5 * np.random.binomial(1, self.temperature, size=self.grid_size**2)
        bonus = bonus.reshape(self.grid_size, self.grid_size)

        malus = -1.0*np.random.binomial(1, self.temperature, size=self.grid_size**2)
        malus = malus.reshape(self.grid_size, self.grid_size)

        self.to_draw = np.zeros((self.max_time + 2, self.grid_size * self.scale,
                                 self.grid_size * self.scale, 3))

        malus[bonus > 0] = 0

        self.board = bonus + malus

        self.position = np.zeros((self.grid_size, self.grid_size))
        self.position[0:2,:]= -1
        self.position[:,0:2] = -1
        self.position[-2:, :] = -1
        self.position[-2:, :] = -1
        self.board[self.x, self.y] = 0
        self.t = 0

        state = np.concatenate((self.board.reshape(self.grid_size, self.grid_size,1),
                                self.position.reshape(self.grid_size, self.grid_size,1)),
                               axis=2)

        state = state[self.x - 2:self.x + 3, self.y - 2:self.y + 3, :]
        return state
```

The following elements are important because they correspond to the hyper parameters for this project:

```python
# parameters
size = 13
T = 200
temperature = 0.3
epochs_train = 21 # set small when debugging
epochs_test = 15 # set small when debugging

# display videos
def display_videos(name):
    video = io.open(name, 'r+b').read()
    encoded = base64.b64encode(video)
    return ''''''.format(encoded.decode('ascii'))
```

__Question 2__ Explain the use of the arrays ```position``` and ```board```.

\begin{itemize}
    \item The position array represents the position of the rat on the grid. One particular aspect is the border: the 2 first / last rows and columns are considered as walls and cannot be access. Therefore, the value of position for these cases are defined as -1. The position where is the rat is valued at 1.
    \item The board array represents the different rewards that the rat can obtain for every cases of the grid. Typically, the board value is equal to 0.5 where there is cheese, equal to -1 where there is poison and 0 otherwise.
\end{itemize}

## Random Agent

***
__Question 3__ Implement a random Agent (only ```learned_act``` needs to be implemented):

```python
class RandomAgent(Agent):
    def __init__(self):
        super(RandomAgent, self).__init__()

    def learned_act(self, s):
        """Return a rabdom action."""

        action = np.random.randint(0, 4, 1)[0]

        return action
```

***
***
__Question 4__ Visualize the game moves. You need to fill in the following function for the evaluation:

```python
def test(agent, env, epochs, prefix='', display=True, save=True):
    """Visualise the moves of the rat"""

    # Number of won games
    score = 0

    for e in range(epochs):

        # Initialisation of win and lose
        win = 0
        lose = 0

        # This assumes that the games will end
        game_over = False

        # At each epoch, we restart to a fresh game and get the initial state
        state = env.reset()

        while not game_over:

            # The agent performs an action
            action = agent.act(state)

            # Apply an action to the environment, get the next state, the reward
            # and if the games end
            prev_state = state
            state, reward, game_over = env.act(action)

            # Update the counters
            if reward > 0:
                win = win + reward
            if reward < 0:
                lose = lose -reward

            # Apply the reinforcement strategy
#             loss = agent.reinforce(prev_state, state, action, reward, game_over)

        # Save as a mp4
        if save:
            env.draw(prefix + str(e))

        # Update stats
        score = score + win-lose

        if display:
            print("Win/lose count {}/{}. Average score ({})"
                  .format(win, lose, score / epochs))

    if display:
        print('Final score: ' + str(score/epochs))

    return score
```

```python
# Initialize the game
env = Environment(grid_size=size, max_time=T, temperature=temperature)

# Initialize the agent!
agent = RandomAgent()

test(agent, env, epochs_test, prefix='./Results/random')
HTML(display_videos('./Results/random0.mp4'))
```

    Win/lose count 10.0/19.0. Average score (-0.6)
    Win/lose count 6.5/22.0. Average score (-1.6333333333333333)
    Win/lose count 7.0/12.0. Average score (-1.9666666666666666)
    Win/lose count 7.0/10.0. Average score (-2.1666666666666665)
    Win/lose count 5.0/13.0. Average score (-2.7)
    Win/lose count 7.0/18.0. Average score (-3.433333333333333)
    Win/lose count 11.5/16.0. Average score (-3.7333333333333334)
    Win/lose count 8.0/14.0. Average score (-4.133333333333334)
    Win/lose count 6.0/6.0. Average score (-4.133333333333334)
    Win/lose count 9.5/16.0. Average score (-4.566666666666666)
    Win/lose count 10.0/11.0. Average score (-4.633333333333334)
    Win/lose count 10.5/14.0. Average score (-4.866666666666666)
    Win/lose count 9.0/16.0. Average score (-5.333333333333333)
    Win/lose count 9.5/8.0. Average score (-5.233333333333333)
    Win/lose count 7.5/11.0. Average score (-5.466666666666667)
    Final score: -5.466666666666667

***
## DQN

Let us assume here that $T=\infty$.

***
__Question 5__ Let $\pi$ be a policy, show that:

\begin{equation*}
Q^{\pi}(s,a)=E_{(s',a')\sim p(.|s,a)}[r(s,a)+\gamma Q^{\pi}(s',a')]
\end{equation*}

Then, show that for the optimal policy $\pi^*$ (we assume its existence), the following holds:

\begin{equation*}
Q^{*}(s,a)=E_{s'\sim \pi^*(.|s,a)}[r(s,a)+\gamma\max_{a'}Q^{*}(s',a')].
\end{equation*}
Finally, deduce that a plausible objective is:

\begin{equation*}
\mathcal{L}(\theta)=E_{s' \sim \pi^*(.|s,a)}\Vert r+\gamma\max\max_{a'}Q(s',a',\theta)-Q(s,a,\theta)\Vert^{2}.
\end{equation*}

\begin{itemize}
    \item The first result represents the Bellman equation. Let's prove it.
    We denote by $(S_t)$ and $(A_t)$ the series of random variables representing the states and action over time. Therefore, if we take up the notations of the slides of the course, it has:
    \begin{align*}
        Q^{\pi}(s_t, a_t) &= \mathbb{E}_{\pi} \left[ \sum_{k=0}^{+\infty} \gamma^k * r(s_{t + k}, a_{t + k}) ~ \big| ~ S_t=s_t, A_t=a_t \right] \\
        &= r(s_t, a_t) + \gamma \mathbb{E}_{\pi} \left[ \sum_{k=0}^{+\infty} \gamma^k * r(s_{(t + 1) + k}, a_{(t + 1) + k}) ~ \big| ~ S_t=s_t, A_t=a_t \right] \\
        &= r(s_t, a_t) + \gamma \mathbb{E}_{\pi} \left[ \sum_{s_{t+1}, a_{t+1}} \mathbf{1}_{ \{ S_{t+1} = s_{t+1}, A_{t+1} = a_{t+1} \} } \underbrace{\mathbb{E}_{\pi} \left[ \sum_{k=0}^{+\infty} \gamma^k * r(s_{(t + 1) + k}, a_{(t + 1) + k}) ~ \big| ~ S_{t+1}=s_{t+1}, A_{t+1}=a_{t+1} \right]}_{ = Q^{\pi}(s_{t+1}, a_{t+1})} ~ \big| ~ S_t=s_t, A_t=a_t \right] \\
        &= r(s_t, a_t) + \gamma \sum_{s_{t+1}, a_{t+1}} \mathbb{P}^{\pi}\left(S_{t+1}=s_{t+1}, A_{t+1}=a_{t+1} ~ | ~ S_t=s_t, A_t=a_t \right) Q^{\pi}(s_{t+1}, a_{t+1}) \\
        &= r(a_t, s_t) + \gamma \mathbb{E}_{(s_{t+1}, a_{t+1}) \sim p(.,. | s_t, a_t)} \left[ Q^{\pi}(s_{t+1}, a_{t+1}) \right]
    \end{align*}
    Moreover, these results are true for every $t$. So, it obtains the result.
    \item This second equation corresponds to the Optimal Bellman Equation. Let's prove it: If it uses again the same equations with $Q^*$, it has:
    \begin{align*}
    Q^{*}(s_t, a_t) &= r(s_t, a_t) + \gamma \sum_{s_{t+1}, a_{t+1}} \mathbb{P}^{\pi^*}\left(S_{t+1}=s_{t+1}, A_{t+1}=a_{t+1} ~ | ~ S_t=s_t, A_t=a_t \right) Q^{*}(s_{t+1}, a_{t+1}) \\
    &= r(s_t, a_t) + \gamma \sum_{s_{t+1}, a_{t+1}} \mathbb{P}\left(S_{t+1}=s_{t+1} ~ | ~ S_t=s_t, A_t=a_t \right) \pi^*(a_{t+1} ~ | ~ S_{t+1} = s_{t+1}) Q^{*}(s_{t+1}, a_{t+1})
    \end{align*}
    But, by definition of the optimal policy,
    \begin{equation*}
        \pi^*(a_{t+1} ~ | ~ S_{t+1} = s_{t+1}) = \left\{ \begin{array}{ cl}
                                                            \frac{1}{|\text{argmax}_{a} Q^{*}(s_{t+1}, a)|}, & \text{if} ~ a_{t+1} \in \text{argmax}_{a} Q^{*}(s_{t+1}, a) \\
                                                            0, & \text{otherwise}
                                                          \end{array}
                                                  \right.
   \end{equation*}
  Therefore:
   \begin{align*}
    Q^{*}(s_t, a_t) &= r(s_t, a_t) + \gamma \sum_{s_{t+1}} \mathbb{P}\left(S_{t+1}=s_{t+1} ~ | ~ S_t=s_t, A_t=a_t \right) max_{a_{t+1}} Q^{*}(s_{t+1}, a_{t+1}) \\
    &= r(a_t, s_t) + \gamma \mathbb{E}_{s_{t+1} \sim p(. | s_t, \pi^*(.|s_t))} \left[ \max_{a_{t+1}} Q^{*}(s_{t+1}, a_{t+1}) \right]
    \end{align*}
    And as before, the result does not depend on the time $t$. It obtains theferore the result.
    \item So,now let suppose that we succeed to parametrised Q by a parameter $\theta$. So, if it is looking for the the optimal policy $\pi^*$, it corresponds to find $\theta$ such that the previous equation on Q is respected. So, we would like to find theta such the two terms of the previous equation are as close as possible. Therefore, we can compute:
    \begin{align*}
        \mathcal{L}_1(\theta) &= \Vert E_{s' \sim \pi^*(.|s,a)} \left[ r + \gamma \max_{a'}Q(s',a',\theta) \right] - Q(s, a, \theta) \Vert^2 \\
        &= \Vert E_{s' \sim \pi^*(.|s,a)} \left[ r + \gamma \max_{a'}Q(s',a',\theta) - Q(s, a, \theta) \right] \Vert^2 \\
        &\leq E_{s' \sim \pi^*(.|s,a)} \left[ \Vert r + \gamma \max_{a'}Q(s',a',\theta) - Q(s, a, \theta) \Vert^2 \right], ~~~ \text{by Jensen inequality} \\
        &\leq \mathcal{L}(\theta)
    \end{align*}
    So $\mathcal{L}(\theta)$ can be used as pausible objective.
\end{itemize}

***
The DQN-learning algorithm relies on these derivations to train the parameters $\theta$ of a Deep Neural Network:

1. At the state $s_t$, select the action $a_t$ with best reward using $Q_t$ and store the results;

2. Obtain the new state $s_{t+1}$ from the environment $p$;

3. Store $(s_t,a_t,s_{t+1})$;

4. Obtain $Q_{t+1}$ by minimizing  $\mathcal{L}$ from a recovered batch from the previously stored results.

***
__Question 6__ Implement the class ```Memory``` that stores moves (in a replay buffer) via ```remember``` and provides a ```random_access``` to these. Specify a maximum memory size to avoid side effects. You can for example use a ```list()``` and set by default ```max_memory=100```.

```python
class Memory(object):
    def __init__(self, max_memory=100):
        self.max_memory = max_memory
        self.memory = list()

    def remember(self, m):
        """Store (s_t, a_t, s_t+1, r_t, game_over_t)."""

        # Current length of the memory
        n = len(self.memory)

        # Append the new state
        if n < self.max_memory:
            self.memory.append(m)
        else:
            # Replace a random state
            idx = np.random.choice(np.arange(n))
            self.memory[idx] = m

    def random_access(self):
        """Extract a random quintuplet (s_t, a_t, s_t+1, r_t, game_over_t) in memory."""

        # Current length of the memory
        n = len(self.memory)

        # Choose a random index
        idx = np.random.choice(np.arange(n))
        quintuplet = self.memory[idx]

        return quintuplet
```

***
The pipeline we will use for training is given below:

```python
def train(agent, env, epoch, prefix='', display=True, save=True):
    """Train the agent on env for the given number of epoch."""

    # Number of won games
    score = 0
    loss = 0

    for e in range(epoch):
        # At each epoch, we restart to a fresh game and get the initial state
        state = env.reset()
        # This assumes that the games will terminate
        game_over = False

        win = 0
        lose = 0

        while not game_over:
            # The agent performs an action
            action = agent.act(state)

            # Apply an action to the environment, get the next state, the reward
            # and if the games end
            prev_state = state
            state, reward, game_over = env.act(action)

            # Update the counters
            if reward > 0:
                win = win + reward
            if reward < 0:
                lose = lose -reward

            # Apply the reinforcement strategy
            loss = agent.reinforce(prev_state, state,  action, reward, game_over)

        # Save as a mp4
        if e % 10 == 0:
            env.draw(prefix + str(e))

        # Update stats
        score += win-lose

        if display:
            print("Epoch {:03d}/{:03d} | Loss {:.4f} | Win/lose count {}/{} ({})"
                  .format(e + 1, epoch, loss, win, lose, win-lose))

        if save:
            agent.save(name_weights=prefix + 'model.h5', name_model=prefix + 'model.json')
```

***
__Question 7__ Implement the DQN training algorithm using a cascade of fully connected layers. You can use different learning rate, batch size or memory size parameters. In particular, the loss might oscillate while the player will start to win the games. You have to find a good criterium.

```python
class DQN(Agent):
    def __init__(self, grid_size,  epsilon=0.1, memory_size=100, batch_size=16, n_state=2):
        """Initialisation of Deep Q Network."""
        super().__init__(epsilon=epsilon)

        # Discount for Q learning
        self.discount = 0.99

        # Grid size
        self.grid_size = grid_size

        # number of state
        self.n_state = n_state

        # Memory
        self.memory = Memory(memory_size)

        # Batch size when learning
        self.batch_size = batch_size

    def learned_act(self, s):
        """Choose the action according the current model."""

        # Reshape the state
        s_t = s.reshape(1, 5, 5, self.n_state)

        # Choose the maximising action
        a_t = np.argmax(self.model.predict(s_t))
        if a_t < 0:
            print(a_t)

        return a_t

    def reinforce(self, s_, n_s_, a_, r_, game_over_):
        """Two steps: first memorize the states, second learn from the pool."""

        # First Step
        self.memory.remember([s_, n_s_, a_, r_, game_over_])

        # Second Step
        input_states = np.zeros((self.batch_size, 5, 5, self.n_state)) #5, self.n_state))
        target_q = np.zeros((self.batch_size, 2), dtype=np.dtype(float, int))

        for i in range(self.batch_size):

            # Sample a random transition
            s_, n_s_, a_, r_, game_over_ = self.memory.random_access()

            # Add this state in input states
            input_states[i, :, :, :] = s_

            if game_over_:
                target_q[i, :] = r_
            else:
                n_s_ = n_s_.reshape(1, 5, 5, self.n_state)
                target_q[i, :] = [r_ + self.discount * np.max(self.model.predict(n_s_)), a_]

        # HINT: Clip the target to avoid exploiding gradients.. -- clipping is a bit tighter
        target_q = np.clip(target_q, -5, 5)
        l = self.model.train_on_batch(input_states, target_q)

        return l

    def save(self, name_weights='model.h5', name_model='model.json'):

        self.model.save_weights(name_weights, overwrite=True)

        with open(name_model, "w") as outfile:
            json.dump(self.model.to_json(), outfile)

    def load(self, name_weights='model.h5', name_model='model.json'):

        with open(name_model, "r") as jfile:
            model = model_from_json(json.load(jfile))

        model.load_weights(name_weights)
        model.compile("sgd", "mse")
        self.model = model

    # Custom loss
    def my_loss(self, data, y_pred):
        """Compute the mse between y_j and Q(s_j, a_j)."""

        # Length
        n = np.shape(y_pred)[0]

        # Extract the corresponding y_pred
        idx = tf.reshape(K.cast(data[:, 1], tf.int32), (-1, 1))
        index = tf.concat([tf.reshape(tf.range(0, self.batch_size), (-1, 1)), idx], axis=1)
        y_pred_2 = tf.gather_nd(y_pred, index)
        y_true = data[:, 0]

        return tf.losses.mean_squared_error(y_true, y_pred_2)

class DQN_FC(DQN):
    def __init__(self, *args, lr=0.1, **kwargs):
        super().__init__(*args, **kwargs)

        # NN Model
        model = Sequential()
        model.add(Flatten(input_shape=(5, 5, self.n_state, )))
        model.add(Dense(32, activation='relu'))
        model.add(Dense(32, activation='relu'))
        model.add(Dense(16, activation='relu'))
        model.add(Dense(4, activation='softmax'))

        model.compile(sgd(lr=lr, decay=1e-4, momentum=0.0), self.my_loss)
        self.model = model

#         print(model.summary())
```

```python
env = Environment(grid_size=size, max_time=T, temperature=0.3)
agent = DQN_FC(size, lr=1, epsilon=0.1, memory_size=1000, batch_size=128)
train(agent, env, epochs_train, prefix='./Results/fc_train')
HTML(display_videos('./Results/fc_train10.mp4'))
```

    Epoch 001/021 | Loss 0.0112 | Win/lose count 1.0/1.0 (0.0)
    Epoch 002/021 | Loss 0.0395 | Win/lose count 4.5/5.0 (-0.5)
    Epoch 003/021 | Loss 0.0241 | Win/lose count 2.0/3.0 (-1.0)
    Epoch 004/021 | Loss 0.0283 | Win/lose count 1.0/2.0 (-1.0)
    Epoch 005/021 | Loss 0.0189 | Win/lose count 6.5/4.0 (2.5)
    Epoch 006/021 | Loss 0.0279 | Win/lose count 1.5/4.0 (-2.5)
    Epoch 007/021 | Loss 0.0224 | Win/lose count 2.5/2.0 (0.5)
    Epoch 008/021 | Loss 0.0142 | Win/lose count 3.5/1.0 (2.5)
    Epoch 009/021 | Loss 0.0127 | Win/lose count 6.0/2.0 (4.0)
    Epoch 010/021 | Loss 0.0163 | Win/lose count 3.5/2.0 (1.5)
    Epoch 011/021 | Loss 0.0182 | Win/lose count 3.5/2.0 (1.5)
    Epoch 012/021 | Loss 0.0198 | Win/lose count 9.0/2.0 (7.0)
    Epoch 013/021 | Loss 0.0130 | Win/lose count 7.0/3.0 (4.0)
    Epoch 014/021 | Loss 0.0215 | Win/lose count 5.5/1.0 (4.5)
    Epoch 015/021 | Loss 0.0187 | Win/lose count 6.0/2.0 (4.0)
    Epoch 016/021 | Loss 0.0351 | Win/lose count 4.0/5.0 (-1.0)
    Epoch 017/021 | Loss 0.0300 | Win/lose count 9.5/2.0 (7.5)
    Epoch 018/021 | Loss 0.0182 | Win/lose count 9.5/2.0 (7.5)
    Epoch 019/021 | Loss 0.0221 | Win/lose count 5.5/2.0 (3.5)
    Epoch 020/021 | Loss 0.0277 | Win/lose count 5.0/2.0 (3.0)
    Epoch 021/021 | Loss 0.0170 | Win/lose count 2.0/2.0 (0.0)

```python
def GridSearch(dqn, train_function=train, env=env):
    """Do a gridsearch on the following parameters: batch_size, memory_size, lr.
       """

    # Possible values of the optimising parameters
    batch_size_l = [64]
    memory_size_l = [1000]
    lr_l = [1, 0.1, 0.01, 0.001]

    # Initialisation of best parameters
    best_parameters = [0, 0, 0]
    max_score = -float("inf")
    i = 0

    # Gridsearch
    for batch_size in batch_size_l:
        for memory_size in memory_size_l:
            for lr in lr_l:

                # Definition of the agent
                agent = dqn(batch_size, memory_size, lr)

                # Training of the agent
                train_function(agent, env, epochs_train, prefix='./Results/fc_train_' + str(i) + "_",
                               display=False, save=False)

                # Testing of the agent
                score_test = test(agent, env, epochs_test,
                                  prefix='./Results/fc_test_' + str(i),
                                  display=False, save=False)

                # Update of best parameters
                if max_score < score_test:
                    max_score = score_test
                    best_parameters = [batch_size, memory_size, lr]
                    print("New best score:", score_test)
                    print("Best parameters:", best_parameters)
                # Update of i
                i += 1
                n = len(batch_size_l) * len(memory_size_l) * len(lr_l)
                print("Step done: ", i / n * 100)

    # Display best parameters
    print(best_parameters)
```

```python
# Definition of the environment
env = Environment(grid_size=size, max_time=T, temperature=0.3)

# Definition of the agent
dqn = lambda batch_size, memory_size, lr: DQN_CNN(size, lr=lr, epsilon=0.1,
                                                  memory_size=memory_size,
                                                  batch_size=batch_size)

# Compute the GridSearch
GridSearch(dqn, env=env)
```

For obtaining the best results, I implemented a GridSearch over different values of the following parameters:
\begin{itemize}
    \item batch_size
    \item memory_size
    \item learning_rate
\end{itemize}
It seems that the two first parameters do not have a big impact for the environment selected. However, it seems that a big learning rate is required.

Also, we implemented the loss in the function "my_loss" which correspond to:
\begin{equation}
    \sum_{i=1}^N (Q(s_i, s_a) - y_i)^2, ~~ \text{with} ~~ y_i = \left\{\begin{array}{cl}
                                                                           r_i & \text{if} ~ \text{game_over} = True \\
                                                                           r_i + \max_{a'} Q({s'}_i, a') & \text{otherwise}
                                                                       \end{array}
                                                                \right.
\end{equation}

***
***
__Question 8__ Implement the DQN training algorithm using a CNN (for example, 2 convolutional layers and one final fully connected layer).

```python
class DQN_CNN(DQN):
    def __init__(self, *args, lr=0.1, **kwargs):
        super().__init__(*args, **kwargs)

        ## CNN
        model = Sequential()
        model.add(Conv2D(16, (3, 3), activation='relu', input_shape=(5, 5, self.n_state, )))
        model.add(Conv2D(16, (3, 3), activation='relu'))
        model.add(Flatten())
        model.add(Dense(4, activation='softmax'))

        model.compile(sgd(lr=lr, decay=1e-4, momentum=0.0), self.my_loss)
        self.model = model
```

```python
env = Environment(grid_size=size, max_time=T, temperature=0.3)
agent = DQN_CNN(size, lr=1, epsilon=0.1, memory_size=1000, batch_size=128)
train(agent,env,epochs_train,prefix='./Results/cnn_train')
HTML(display_videos('./Results/cnn_train10.mp4'))
```

    Epoch 001/021 | Loss 0.0390 | Win/lose count 3.5/4.0 (-0.5)
    Epoch 002/021 | Loss 0.0572 | Win/lose count 3.5/3.0 (0.5)
    Epoch 003/021 | Loss 0.0604 | Win/lose count 2.5/2.0 (0.5)
    Epoch 004/021 | Loss 0.0165 | Win/lose count 3.0/2.0 (1.0)
    Epoch 005/021 | Loss 0.0126 | Win/lose count 2.5/6.0 (-3.5)
    Epoch 006/021 | Loss 0.0129 | Win/lose count 1.0/4.0 (-3.0)
    Epoch 007/021 | Loss 0.0187 | Win/lose count 3.5/1.0 (2.5)
    Epoch 008/021 | Loss 0.0166 | Win/lose count 6.5/5.0 (1.5)
    Epoch 009/021 | Loss 0.0229 | Win/lose count 6.0/4.0 (2.0)
    Epoch 010/021 | Loss 0.0130 | Win/lose count 3.0/1.0 (2.0)
    Epoch 011/021 | Loss 0.0360 | Win/lose count 5.0/1.0 (4.0)
    Epoch 012/021 | Loss 0.0171 | Win/lose count 5.5/3.0 (2.5)
    Epoch 013/021 | Loss 0.0170 | Win/lose count 5.5/2.0 (3.5)
    Epoch 014/021 | Loss 0.0159 | Win/lose count 4.0/3.0 (1.0)
    Epoch 015/021 | Loss 0.0190 | Win/lose count 2.5/2.0 (0.5)
    Epoch 016/021 | Loss 0.0221 | Win/lose count 7.0/5.0 (2.0)
    Epoch 017/021 | Loss 0.0235 | Win/lose count 2.5/4.0 (-1.5)
    Epoch 018/021 | Loss 0.0245 | Win/lose count 7.0/2.0 (5.0)
    Epoch 019/021 | Loss 0.0199 | Win/lose count 4.5/1.0 (3.5)
    Epoch 020/021 | Loss 0.0204 | Win/lose count 3.0/4.0 (-1.0)
    Epoch 021/021 | Loss 0.0256 | Win/lose count 9.0/0 (9.0)

***
***
__Question 9__ Test both algorithms and compare their performances. Which issue(s) do you observe? Observe also different behaviors by changing the temperature.

```python
env = Environment(grid_size=size, max_time=T, temperature=0.1)
agent_cnn = DQN_CNN(size, lr=1, epsilon=0.1, memory_size=1000, batch_size=128)
agent_cnn.load(name_weights='./Results/cnn_trainmodel.h5',
               name_model='./Results/cnn_trainmodel.json')

agent_fc = DQN_FC(size, lr=1, epsilon=0.1, memory_size=1000, batch_size=128)
agent_cnn.load(name_weights='./Results/fc_trainmodel.h5',
               name_model='./Results/fc_trainmodel.json')
print('Test of the CNN')
test(agent_cnn, env, epochs_test, prefix='./Results/cnn_test')
print('\nTest of the FC')
test(agent_fc, env, epochs_test, prefix='./Results/fc_test');
```

    Test of the CNN
    Win/lose count 3.0/3.0. Average score (0.0)
    Win/lose count 3.0/0. Average score (0.2)
    Win/lose count 2.0/2.0. Average score (0.2)
    Win/lose count 2.5/2.0. Average score (0.23333333333333334)
    Win/lose count 2.5/1.0. Average score (0.3333333333333333)
    Win/lose count 2.5/1.0. Average score (0.43333333333333335)
    Win/lose count 2.0/4.0. Average score (0.3)
    Win/lose count 2.5/1.0. Average score (0.4)
    Win/lose count 2.5/1.0. Average score (0.5)
    Win/lose count 3.0/1.0. Average score (0.6333333333333333)
    Win/lose count 1.0/1.0. Average score (0.6333333333333333)
    Win/lose count 3.0/3.0. Average score (0.6333333333333333)
    Win/lose count 0.5/0. Average score (0.6666666666666666)
    Win/lose count 3.5/1.0. Average score (0.8333333333333334)
    Win/lose count 3.0/0. Average score (1.0333333333333334)
    Final score: 1.0333333333333334

    Test of the FC
    Win/lose count 2.0/0. Average score (0.13333333333333333)
    Win/lose count 1.0/1.0. Average score (0.13333333333333333)
    Win/lose count 2.0/1.0. Average score (0.2)
    Win/lose count 1.5/2.0. Average score (0.16666666666666666)
    Win/lose count 1.5/1.0. Average score (0.2)
    Win/lose count 0/4.0. Average score (-0.06666666666666667)
    Win/lose count 4.0/4.0. Average score (-0.06666666666666667)
    Win/lose count 0.5/0. Average score (-0.03333333333333333)
    Win/lose count 0.5/1.0. Average score (-0.06666666666666667)
    Win/lose count 2.5/6.0. Average score (-0.3)
    Win/lose count 0.5/3.0. Average score (-0.4666666666666667)
    Win/lose count 3.0/5.0. Average score (-0.6)
    Win/lose count 1.5/3.0. Average score (-0.7)
    Win/lose count 0.5/6.0. Average score (-1.0666666666666667)
    Win/lose count 1.0/0. Average score (-1.0)
    Final score: -1.0

```python
HTML(display_videos('./Results/cnn_test10.mp4'))
```

```python
HTML(display_videos('./Results/fc_test10.mp4'))
```

For optimised parameters, we observed:
\begin{itemize}
    \item The performance of the DQN with Convolutionnal networks performs in average better than the Fully Connected one.
    \item Also, we can observe that the agent get easily blocked in the corners and only explore one border There is a lack of exploration here.
    \item When the temperature increases, the score of the two agents increase also a lot. Indeed, there is much more food in the environment so it is easy for the agent to increase its score.
\end{itemize}

***

The algorithm tends to not explore the map which can be an issue. We propose two ideas in order to encourage exploration:
1. Incorporating a decreasing $\epsilon$-greedy exploration. You can use the method ```set_epsilon```
2. Append via the environment a new state that describes if a cell has been visited or not

***
__Question 10__ Design a new ```train_explore``` function and environment class ```EnvironmentExploring``` to tackle the issue of exploration.

```python
def train_explore(agent, env, epoch, prefix='', display=True, save=True):
    """Train the agent to explore env for the given number of epoch."""

    # Number of won games
    score = 0
    loss = 0

    for e in range(epoch):

        # At each epoch, we restart to a fresh game and get the initial state
        state = env.reset()
        # This assumes that the games will terminate
        game_over = False

        win = 0
        lose = 0

        # Definition of epsilon
        epsilon = 1 / (1 + e)**2
        agent.set_epsilon(epsilon)

        while not game_over:
            # The agent performs an action
            action = agent.act(state)

            # Apply an action to the environment, get the next state, the reward
            # and if the games end
            prev_state = state
            state, reward, game_over = env.act(action)

            # Update the counters
            if reward > 0:
                win = win + reward
            if reward < 0:
                lose = lose -reward

            # Apply the reinforcement strategy
            loss = agent.reinforce(prev_state, state,  action, reward, game_over)

        # Save as a mp4
        if e % 10 == 0:
            env.draw(prefix + str(e))

        # Update stats
        score += win-lose

        if display:
            print("Epoch {:03d}/{:03d} | Loss {:.4f} | Win/lose count {}/{} ({})"
                  .format(e + 1, epoch, loss, win, lose, win-lose))

        if save:
            agent.save(name_weights=prefix + 'model.h5', name_model=prefix + 'model.json')

class EnvironmentExploring(Environment):
    def __init__(self, grid_size=10, max_time=500, temperature=0.1, train_binary=True):
        super().__init__(grid_size=grid_size, max_time=max_time,
                         temperature=temperature)
        self.train_binary = train_binary

    def act(self, action):
        """This function returns the new state, reward and decides if the
        game ends."""

        self.get_frame(int(self.t))

        self.position = np.zeros((self.grid_size, self.grid_size))

        self.position[0:2, :]= -1
        self.position[:, 0:2] = -1
        self.position[-2:, :] = -1
        self.position[-2:, :] = -1

        self.position[self.x, self.y] = 1

        if action == 0: # Going up (or down on the image)
            if self.x == self.grid_size - 3:
                self.x = self.x - 1
            else:
                self.x = self.x + 1
        elif action == 1: # Going up on the image
            if self.x == 2:
                self.x = self.x + 1
            else:
                self.x = self.x - 1
        elif action == 2: # Going right on the image
            if self.y == self.grid_size - 3:
                self.y = self.y - 1
            else:
                self.y = self.y + 1
        elif action == 3: # Going left on the image
            if self.y == 2:
                self.y = self.y + 1
            else:
                self.y = self.y - 1
        else:
            RuntimeError('Error: action not recognized')

        self.t = self.t + 1

        # Update of the reward
        reward = 0
        if self.train_binary:
            reward = -self.malus_position[self.x, self.y]
        self.malus_position[self.x, self.y] = 0.1
        reward = reward + self.board[self.x, self.y]
        self.board[self.x, self.y] = 0

        game_over = self.t > self.max_time
        state = np.concatenate((self.malus_position.reshape(self.grid_size, self.grid_size,1),
                                self.board.reshape(self.grid_size, self.grid_size,1),
                                self.position.reshape(self.grid_size, self.grid_size,1)),
                               axis=2)
        state = state[self.x-2:self.x+3, self.y-2:self.y+3, :]

        return state, reward, game_over

    def reset(self):
        """This function resets the game and returns the initial state"""

        self.x = np.random.randint(3, self.grid_size-3, size=1)[0]
        self.y = np.random.randint(3, self.grid_size-3, size=1)[0]

        # Definition of board
        bonus = 0.5 * np.random.binomial(1, self.temperature, size=self.grid_size**2)
        bonus = bonus.reshape(self.grid_size, self.grid_size)

        malus = -1.0*np.random.binomial(1, self.temperature, size=self.grid_size**2)
        malus = malus.reshape(self.grid_size, self.grid_size)

        self.to_draw = np.zeros((self.max_time + 2, self.grid_size * self.scale,
                                 self.grid_size * self.scale, 3))

        malus[bonus > 0] = 0

        self.board = bonus + malus

        # Definition of malus_position
        self.malus_position = np.zeros((self.grid_size, self.grid_size))

        # Definition of position
        self.position = np.zeros((self.grid_size, self.grid_size))
        self.position[0:2,:]= -1
        self.position[:,0:2] = -1
        self.position[-2:, :] = -1
        self.position[-2:, :] = -1
        self.board[self.x, self.y] = 0
        self.t = 0

        state = np.concatenate((self.malus_position.reshape(self.grid_size, self.grid_size, 1),
                                self.board.reshape(self.grid_size, self.grid_size,1),
                                self.position.reshape(self.grid_size, self.grid_size,1)),
                               axis=2)

        state = state[self.x - 2:self.x + 3, self.y - 2:self.y + 3, :]
        return state
```

```python
# Definition of the environment
env = EnvironmentExploring(grid_size=size, max_time=T, temperature=0.3)

# Definition of the agent
dqn = lambda batch_size, memory_size, lr: DQN_CNN(size, lr=lr, epsilon=0.1,
                                                  memory_size=memory_size,
                                                  batch_size=batch_size, n_state=3)

# Compute the GridSearch
GridSearch(dqn, train_function=train_explore, env=env)
```

```python
# Training
env = EnvironmentExploring(grid_size=size, max_time=T, temperature=0.3)
agent = DQN_CNN(size, lr=1, memory_size=1000, batch_size=128, n_state=3)
train_explore(agent, env, epochs_train, prefix='cnn_train_explore')
HTML(display_videos('cnn_train_explore20.mp4'))
```

    Epoch 001/021 | Loss 0.0510 | Win/lose count 5.0/33.700000000000124 (-28.700000000000124)
    Epoch 002/021 | Loss 0.0472 | Win/lose count 10.0/24.20000000000007 (-14.20000000000007)
    Epoch 003/021 | Loss 0.0386 | Win/lose count 11.0/18.70000000000001 (-7.70000000000001)
    Epoch 004/021 | Loss 0.0341 | Win/lose count 11.0/18.59999999999998 (-7.59999999999998)
    Epoch 005/021 | Loss 0.0354 | Win/lose count 14.5/18.700000000000014 (-4.2000000000000135)
    Epoch 006/021 | Loss 0.0370 | Win/lose count 7.5/19.4 (-11.899999999999999)
    Epoch 007/021 | Loss 0.0354 | Win/lose count 16.5/14.499999999999972 (2.0000000000000284)
    Epoch 008/021 | Loss 0.0419 | Win/lose count 15.5/15.999999999999966 (-0.49999999999996625)
    Epoch 009/021 | Loss 0.0323 | Win/lose count 9.5/16.29999999999996 (-6.799999999999962)
    Epoch 010/021 | Loss 0.0358 | Win/lose count 6.5/18.599999999999994 (-12.099999999999994)
    Epoch 011/021 | Loss 0.0414 | Win/lose count 5.5/17.799999999999983 (-12.299999999999983)
    Epoch 012/021 | Loss 0.0295 | Win/lose count 21.0/16.199999999999964 (4.800000000000036)
    Epoch 013/021 | Loss 0.0297 | Win/lose count 3.0/18.499999999999993 (-15.499999999999993)
    Epoch 014/021 | Loss 0.0372 | Win/lose count 5.5/20.8 (-15.3)
    Epoch 015/021 | Loss 0.0199 | Win/lose count 14.0/15.199999999999962 (-1.199999999999962)
    Epoch 016/021 | Loss 0.0393 | Win/lose count 17.0/14.999999999999966 (2.0000000000000338)
    Epoch 017/021 | Loss 0.0439 | Win/lose count 15.5/17.39999999999999 (-1.8999999999999915)
    Epoch 018/021 | Loss 0.0255 | Win/lose count 9.0/17.99999999999999 (-8.99999999999999)
    Epoch 019/021 | Loss 0.0369 | Win/lose count 13.5/15.399999999999961 (-1.8999999999999613)
    Epoch 020/021 | Loss 0.0318 | Win/lose count 18.0/14.599999999999964 (3.400000000000036)
    Epoch 021/021 | Loss 0.0215 | Win/lose count 11.0/17.699999999999985 (-6.699999999999985)

```python
# Definition of the environment without training
env = EnvironmentExploring(grid_size=size, max_time=T, temperature=0.3, train_binary=False)

# Evaluation
test(agent, env, epochs_test,prefix='./Results/cnn_test_explore')
HTML(display_videos('./Results/cnn_test_explore14.mp4'))
```

    Win/lose count 5.0/0. Average score (0.3333333333333333)
    Win/lose count 7.0/1.0. Average score (0.7333333333333333)
    Win/lose count 10.5/0. Average score (1.4333333333333333)
    Win/lose count 12.5/0. Average score (2.2666666666666666)
    Win/lose count 6.0/0. Average score (2.6666666666666665)
    Win/lose count 1.5/0. Average score (2.7666666666666666)
    Win/lose count 10.0/1.0. Average score (3.3666666666666667)
    Win/lose count 1.0/0. Average score (3.433333333333333)
    Win/lose count 9.5/0. Average score (4.066666666666666)
    Win/lose count 1.0/0. Average score (4.133333333333334)
    Win/lose count 4.5/0. Average score (4.433333333333334)
    Win/lose count 9.0/0. Average score (5.033333333333333)
    Win/lose count 6.5/0. Average score (5.466666666666667)
    Win/lose count 3.0/0. Average score (5.666666666666667)
    Win/lose count 11.0/0. Average score (6.4)
    Final score: 6.4

***
***
__BONUS question__ Use the expert DQN from the previous question to generate some winning games. Train a model that mimicks its behavior. Compare the performances.

***


https://github.com/heart-group-illinois/COVID19-India

\begin{align}
    \mathcal{C}(i,j; \kappa) := \frac{1}{T}\sum_{t=1}^T \left( \rho^i_t - \langle \rho^i\rangle \right) \left( \rho^j_{t+\kappa} - \langle \rho^j\rangle \right),
\end{align}

```R
library(tidyverse)
library(readr)
library(magrittr)
```

    -- Attaching packages ------------------------------------------------------------------------------------------------------------ tidyverse 1.3.1 --

    v ggplot2 3.3.3     v purrr   0.3.4
    v tibble  3.1.2     v dplyr   1.0.6
    v tidyr   1.1.3     v stringr 1.4.0
    v readr   1.4.0     v forcats 0.5.1

    -- Conflicts --------------------------------------------------------------------------------------------------------------- tidyverse_conflicts() --
    x dplyr::filter() masks stats::filter()
    x dplyr::lag()    masks stats::lag()

    Attaching package: 'magrittr'

    The following object is masked from 'package:purrr':

        set_names

    The following object is masked from 'package:tidyr':

        extract

```R
version
```

                   _
    platform       x86_64-w64-mingw32
    arch           x86_64
    os             mingw32
    system         x86_64, mingw32
    status
    major          4
    minor          1.0
    year           2021
    month          05
    day            18
    svn rev        80317
    language       R
    version.string R version 4.1.0 (2021-05-18)
    nickname       Camp Pontanezen

```R
df_All <- read_csv("Data/dfIndia_to_models_districts_with_Mobility_for_paper.csv")
```

    -- Column specification -----------------------------------------------------------------------------------------------------------------------------
    cols(
      .default = col_double(),
      State = col_character(),
      District = col_character(),
      Date = col_date(format = "")
    )
    i Use `spec()` for the full column specifications.

```R
df_All %>% ggplot(aes(Date, y = Mob, colour = District_ID)) +
  geom_line(size = 1)
```


# This is a test sample for Jupyter Notebook.

```python
print("Hello Jupyter Notebook")
```

    Hello Jupyter Notebook

## Hello JNB ver 1.0

```python
print "Hello World"
```

# LaTeX Demo

$$e^x=\sum_{i=0}^\infty \frac{1}{i!}x^i$$

```latex
%%latex

\begin{align}
\dot{x} & = \sigma(y^2-x^2) \\
\dot{y} & = \rho x - y_1 - xz \\
\dot{z} & = -\beta z + xy
\end{align}

\begin{align}
\\
\end{align}

\begin{equation*}
\mathbf{V}_1 \times \mathbf{V}_2 =  \begin{vmatrix}
\mathbf{i} & \mathbf{j} & \mathbf{k} \\
\frac{\partial X}{\partial u} &  \frac{\partial Y}{\partial u} & 0 \\
\frac{\partial X}{\partial v} &  \frac{\partial Y}{\partial v} & 0
\end{vmatrix}
\end{equation*}

$$
$$

\begin{eqnarray}
x' &=& &x \sin\phi &+& z \cos\phi \\
z' &=& - &x \cos\phi &+& z \sin\phi \\
\end{eqnarray}
```

\begin{align}
\dot{x} & = \sigma(y^2-x^2) \\
\dot{y} & = \rho x - y_1 - xz \\
\dot{z} & = -\beta z + xy
\end{align}

\begin{align}
\\
\end{align}

\begin{equation*}
\mathbf{V}_1 \times \mathbf{V}_2 =  \begin{vmatrix}
\mathbf{i} & \mathbf{j} & \mathbf{k} \\
\frac{\partial X}{\partial u} &  \frac{\partial Y}{\partial u} & 0 \\
\frac{\partial X}{\partial v} &  \frac{\partial Y}{\partial v} & 0
\end{vmatrix}
\end{equation*}

$$
$$

\begin{eqnarray}
x' &=& &x \sin\phi &+& z \cos\phi \\
z' &=& - &x \cos\phi &+& z \sin\phi \\
\end{eqnarray}

# Magic

```python
%lsmagic
```

    Available line magics:
    %alias  %alias_magic  %autoawait  %autocall  %automagic  %autosave  %bookmark  %cat  %cd  %clear  %colors  %conda  %config  %connect_info  %cp  %debug  %dhist  %dirs  %doctest_mode  %ed  %edit  %env  %gui  %hist  %history  %killbgscripts  %ldir  %less  %lf  %lk  %ll  %load  %load_ext  %loadpy  %logoff  %logon  %logstart  %logstate  %logstop  %ls  %lsmagic  %lx  %macro  %magic  %man  %matplotlib  %mkdir  %more  %mv  %notebook  %page  %pastebin  %pdb  %pdef  %pdoc  %pfile  %pinfo  %pinfo2  %pip  %popd  %pprint  %precision  %prun  %psearch  %psource  %pushd  %pwd  %pycat  %pylab  %qtconsole  %quickref  %recall  %rehashx  %reload_ext  %rep  %rerun  %reset  %reset_selective  %rm  %rmdir  %run  %save  %sc  %set_env  %store  %sx  %system  %tb  %time  %timeit  %unalias  %unload_ext  %who  %who_ls  %whos  %xdel  %xmode

    Available cell magics:
    %%!  %%HTML  %%SVG  %%bash  %%capture  %%debug  %%file  %%html  %%javascript  %%js  %%latex  %%markdown  %%perl  %%prun  %%pypy  %%python  %%python2  %%python3  %%ruby  %%script  %%sh  %%svg  %%sx  %%system  %%time  %%timeit  %%writefile

    Automagic is ON, % prefix IS NOT needed for line magics.

## %alias

```python
%alias A2B echo %s is speaking to %s.
%A2B Vincent Leo

!echo "================= This is a magnificent separate line ================="

%alias ISAY echo \"%l\", I said.
%ISAY Stop Laughing! Idiot.
%ISAY 别笑了，白痴。

!echo "================= This is a magnificent separate line ================="

ISAY?
```

    Vincent is speaking to Leo.
    ================= This is a magnificent separate line =================
    "Stop Laughing! Idiot.", I said.
    "别笑了，白痴。", I said.
    ================= This is a magnificent separate line =================

```python
%pwd
```

    '/home/vincent/pythonws/jnb-sample/test'

```python
%timeit
%sx ls -l
```

    ['total 92',
     '-rw-r--r-- 1 vincent vincent  6694 6月   2 03:36 lottery_test.ipynb',
     '-rw-r--r-- 1 vincent vincent 22660 6月   8 22:10 nb_test.ipynb',
     '-rw-r--r-- 1 vincent vincent  1592 6月   8 22:11 nb_widget_test.ipynb',
     '-rw-r--r-- 1 vincent vincent  6443 6月   8 17:16 py_basic_test.ipynb',
     '-rw-r--r-- 1 vincent vincent  3543 6月   7 22:53 py_class_test.ipynb',
     '-rw-r--r-- 1 vincent vincent  6726 6月   7 01:10 py_function_file_test.ipynb',
     '-rw-r--r-- 1 vincent vincent  4271 6月   5 18:47 py_list_test.ipynb',
     '-rw-r--r-- 1 vincent vincent  8405 6月   5 02:59 py_random_test.ipynb',
     '-rw-r--r-- 1 vincent vincent  3204 6月   8 02:17 py_re_test.ipynb',
     '-rw-r--r-- 1 vincent vincent  6921 6月   6 01:50 py_time_test.ipynb',
     '-rw-rw-r-- 1 vincent vincent    25 6月   7 01:09 sample.txt']

```python
%matplotlib --list
```

    Available matplotlib backends: ['tk', 'gtk', 'gtk3', 'wx', 'qt4', 'qt5', 'qt', 'osx', 'nbagg', 'notebook', 'agg', 'svg', 'pdf', 'ps', 'inline', 'ipympl', 'widget']

## Running javascript in notebook cell.

```python
%%js

if (confirm('This is a javascript sample.\n\nPlease click either one button below.')) {
    console.log('You chose OK!');           // check this out in browser web console by convention in the sub menu 'Web Developer' in Firefox.
    alert('HELLO, Greets from javascript.');
} else {
    console.log('You chose CANCEL!');  // check this out in browser web console by convention in the sub menu 'Web Developer' in Firefox.
    alert('SORRY! You chose CANCEL!');
}
```

    <IPython.core.display.Javascript object>