```python
%matplotlib inline
```

```python
# configure matplotlib
import matplotlib as mpl
mpl.rcParams['text.usetex'] = True

import collections

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns

from chmp.ds import reload
reload('chmp.ds')

from chmp.ds import (
    colorize,
    mpl_set,
    pd_has_ordered_assign,
    pgm,
    plot_gaussian_contour,
)
```

```python
assert pd_has_ordered_assign(), "notebook requires .assign(...) to respect argument order"
```

# d-seperation

Based on Daphne Koller and Nir Friedman "Probabilistic Graphical Models" (2009).

```python
def _plot(edge1_lr, edge2_lr, observed, blocked):
    ec = 'r' if blocked else 'k'

    res = (
        pgm()
        .node('x', '', 1, 1)
        .node('z', '', 3, 1)
        .node('y', '', 2, 1, observed=observed, edgecolor=ec)
    )
    res = res.edge('x', 'y') if edge1_lr else res.edge('y', 'x')
    res = res.edge('y', 'z') if edge2_lr else res.edge('z', 'y')

    return res

_, ((ax11, ax12, ax13), (ax21, ax22, ax23)) = plt.subplots(2, 3, figsize=(16, 2))

_plot(True, True, False, False).render(ax=ax11)
_plot(True, True, True, True).render(ax=ax21)

_plot(False, True, False, False).render(ax=ax12)
_plot(False, True, True, True).render(ax=ax22)

_plot(True, False, False, True).render(ax=ax13)
_plot(True, False, True, False).render(ax=ax23)

pass
```

Visualize the indendence problems by approximating the mean absolute correlation (based on binning for $z$):

$$
    MAC = \frac{\sum_z |\mathrm{corr}(x, y|z)|}{\sum_z}
$$

```python
def compute_macs(df, x, y, conditions):
    result = collections.OrderedDict()

    for condition in conditions:
        if condition:
            label = '| {}'.format(', '.join(condition))

        else:
            label = ''

        key = r'$\mathrm{{MAC}}({}, {}{})$'.format(x, y, label)
        mac = df.pipe(mean_absolute_correlation, x, y, condition=condition)
        result[key] = mac

    return pd.Series(result)

def mean_absolute_correlation(df, x, y, condition=()):
    by = []

    for var in condition:
        by += [pd.qcut(df[var], 11)]

    if by:
        res = df.groupby(by)[[x, y]].corr().unstack()[x, y]
        return abs(res).mean()

    else:
        return abs(df[[x, y]].corr().iloc[0, 1])
```

```python
base_graph = pgm().node('x', 1, 2).node('y', 2, 2).node('z', 1.5, 1)
```

```python
_, (ax1, ax2, ax3, ax4) = plt.subplots(1, 4, figsize=(16, 3.25))

base_graph.edges('z', 'xy').render(title='fork', ax=ax1)

df = pd.DataFrame().assign(
    z=lambda df: np.random.normal(size=10_000),
    x=lambda df: np.random.normal(df["z"]),
    y=lambda df: np.random.normal(df["z"]),
)

df.plot.scatter("x", "y", marker=".", ax=ax2, alpha=0.3)
df.pipe(plot_gaussian_contour, "x", "y", ax=ax2)

for color, (_, group) in colorize(df.groupby(pd.qcut(df["z"], 11))):
    group.sample(n=100).plot.scatter("x", "y", marker=".", ax=ax3, color=color, alpha=0.3)
    group.pipe(plot_gaussian_contour, 'x', 'y', edgecolor=color, ax=ax3)

compute_macs(df, 'x', 'y', [[], ['z']]).plot.bar(ax=ax4, rot=0)
pass
```

```python
_, (ax1, ax2, ax3, ax4) = plt.subplots(1, 4, figsize=(16, 3.25))

base_graph.edges('xy', 'z').render(title='collider', ax=ax1)

df = pd.DataFrame().assign(
    x=lambda df: np.random.normal(size=10_000),
    y=lambda df: np.random.normal(size=10_000),
    z=lambda df: np.random.normal(df["x"] + df["y"]),
)

df.plot.scatter("x", "y", marker=".", ax=ax2, alpha=0.3)
df.pipe(plot_gaussian_contour, "x", "y", ax=ax2)

for color, (_, group) in colorize(df.groupby(pd.qcut(df["z"], 11))):
    group.sample(n=100).plot.scatter("x", "y", marker=".", ax=ax3, color=color, alpha=0.3)
    group.pipe(plot_gaussian_contour, 'x', 'y', edgecolor=color, ax=ax3)

compute_macs(df, 'x', 'y', [[], ['z']]).plot.bar(ax=ax4, rot=0)
pass
```

```python
_, (ax1, ax2, ax3, ax4) = plt.subplots(1, 4, figsize=(16, 3.25))

(
    pgm()
    .node('x', 1, 2)
    .node('y', 2, 2)
    .node('z', 1.5, 1)
    .node('w', 1.5, 0)
    .edge('x', 'z')
    .edge('y', 'z')
    .edge('z', 'w')
    .render(title='collider with one child', ax=ax1)
)

df = pd.DataFrame().assign(
    x=lambda df: np.random.normal(size=10_000),
    y=lambda df: np.random.normal(size=10_000),
    z=lambda df: np.random.normal(df["x"] + df["y"]),
    w=lambda df: np.random.normal(df["z"])
)

sns.heatmap(df.corr(), annot=True, fmt='.2f', ax=ax2)
mpl_set(ax=ax2, invert='y')

for color, (_, group) in colorize(df.pipe(lambda df: df.groupby(pd.qcut(df['z'], 11)))):
    group.sample(n=150).plot.scatter('x', 'y', alpha=0.3, marker='.', color=color, ax=ax3)
    group.pipe(plot_gaussian_contour, 'x', 'y', edgecolor=color, ax=ax3)

compute_macs(df, 'x', 'y', [[], ['z'], ['w'], ['z', 'w']]).plot.bar(ax=ax4, rot=30)
pass
```

# Causal calculus

Based on Judea Pearl "Causal diagrams for empirical research" (1995).

Define:

- $(\dots)_{\overline{X}}$: the expression $\dots$ evaluated in the graph with with all edges that point into $X$ removed
- $(\dots)_{\underline{X}}$: the expression $\dots$ evaluated in graph with all edges that point out of $X$ removed
- $\mathrm{pa}(X)$: the parents of node set $X$
- $\mathrm{an}(X)$: the ancestors of node set $X$

**Rule 1.** If removing edges into $X$, make $Y$ and $Z$ independent, then the conditional probabilities in the causal graph reflect this fact:

$$
\begin{align}
    P(Y|\mathrm{do}(X), Z, W) &= P(Y|\mathrm{do}(X), W)
    & &\text{if $(Y \perp Z|X, W)_\overline{X}$}
\end{align}
$$

**Rule 2.** If $Z$ only acts "forward", i.e., there is no backdoor path, then conditioning and intervening have the same effect:

$$
\begin{align}
    P(Y|\mathrm{do}(X), \mathrm{do}(Z), W) &= P(Y|\mathrm{do}(X), Z, W)
    & & \text{if $(Y \perp Z|X, W)_{\overline{X}, \underline{Z}}$}
\end{align}
$$

**Rule 3.**

$$
\begin{align}
    P(Y|\mathrm{do}(X), \mathrm{do}(Z), W) &= P(Y|\mathrm{do}(X), W)
    & & \text{if $(Y \perp Z|X, W)_{\overline{X}, \overline{Z - \mathrm{an}(W)}}$}
\end{align}
$$

Notable special cases:

$$
\begin{align}
    P(Y|\mathrm{do}(Z)) &= P(Y)
    & & \text{if $(Y \perp Z)_\overline{Z}$, (III)} \\
    P(Y|\mathrm{do}(\mathrm{pa}(Y))) &= P(Y|\mathrm{pa}(Y))
    & & \text{since $(Y \perp \mathrm{pa}(Y))_\underline{\mathrm{pa}(Y)}, (II)$}
\end{align}
$$

The goal is always to remove all instances of $\mathrm{do}(\dots)$ from the expressions using aboves rules.

# Example I

```python
graph = (
    pgm()
    .node('X', 1.25, 1)
    .node('Z', 2.0, 1.75)
    .node('Y', 2.75, 1)
    .edges('Z', 'XY')
    .edges('X', 'Y')
)

_, (ax1, ax2, ax3) = plt.subplots(1, 3, figsize=(12, 3.25))
graph.render(ax=ax1, title='Full graph $\mathcal{G}$')
graph.remove(outgoing='X').render(ax=ax2, title=r'Modified graph: $\mathcal{G}_{\underline{X}}$')
graph.remove(incoming='X').render(ax=ax3, title=r'Modified graph: $\mathcal{G}_{\overline{X}}$')
```

$$
\begin{align}
    p(y|z, \mathrm{do}(x)) &= p(y|z, x)
    && \text{(II), since $(Y \perp X|Z)_\underline{X}$}
\\
    p(z|\mathrm{do}(x)) &= p(z)
    && \text{(III), since $(Z \perp X)_\overline{X}$}
\\
    p(y|\mathrm{do}(x)) &= \sum_z p(y|z, \mathrm{do}(x)) p(z|\mathrm{do}(x))
    &&
\\
    &= \sum_z p(y|z, x) p(z)
    &&
\end{align}
$$

## Example II

```python
graph = (
    pgm()
    .node('E', 1, 2.5)
    .node('X', 1.25, 1)
    .node('Z', 2.0, 1.75)
    .node('A', 3, 2.5)
    .node('Y', 2.75, 1)
    .edges('E', 'XZ')
    .edges('Z', 'XY')
    .edges('A', 'YZ')
    .edges('X', 'Y')
)

_, (ax1, ax2, ax3) = plt.subplots(1, 3, figsize=(16, 4))
graph.render(ax=ax1, title='Full graph $\mathcal{G}$')
graph.remove(incoming='X').render(ax=ax2, title=r'Modified graph: $\mathcal{G}_{\overline{X}}$')
graph.remove(outgoing='X').render(ax=ax3, title=r'Modified graph: $\mathcal{G}_{\underline{X}}$')
```

Application

$$
\begin{align}
    P(Y|\mathrm{do}(X)) &= \sum_{E, Z} P(Y|E, Z, \mathrm{do}(X)) P(E, Z|\mathrm{do}(X))
    &&
\\
\hline
\\
    P(E, Z|\mathrm{do}(X)) &= P(E, Z)
    && \text{(III), since $(\{E, Z\} \perp X)_\overline{X}$}
\\
    P(Y|E, Z, \mathrm{do}(X)) &= P(Y|E, Z, X)
    && \text{(II), since $(Y \perp X|E, Z)_\underline{X}$}
\\
\hline
\\
    &= \sum_{E, Z} P(Y|E, Z, X) P(E, Z)
    &&
\\
    &= \sum_{E, Z} \frac{P(Y, E, Z, X)}{P(X|E, Z)}
\end{align}
$$

# Counterfactuals

Answers to the question: given I used $X = x$ and the path and observed $Y = y$, what whould $Y$ have been had I used $X = x^\prime$.

Can be answered by three steps (Pearl, Glymour, Jewell "Causal inference in statistics - a primer":

- **Abduciton:** determine the distribution of latents given the evedince: $P(U|E = e)$
- **Action:** modify the model, by replacing the equations for $X$ with $X = x^\prime$
- **Prediction:** use the modified model and $P(U|E=e)$ to predict the consequence of the counterfactual

```python

```


# Replication exercise for [Nevo (2000)](#nevo_2000)

This notebook replicates "A Practitioner's Guide to Estimation of Random Coefficients Logit Models of Demand" by Aviv Nevo. This paper is a classic introduction to structural models. I recommend checking the appendix to the paper as well to understand the replication exercise. The code is done in Python.

*By Carlos Felipe Balcazar (NYU).*

## Introduction

The intuition behind the exercise is straightforward: Consider that the law of supply and demand states that demand decreases with price if the good is normal. The problem is that demand does not take place in isolation, demand invitably brings-in supply. Equilibrium prices thus depend on product characteristics, on location, on consumer information, tastes and many other attributes of markets. In equilibrium, prices clear the market on the basis of many elements that are often unobservable to the researcher, therefore any attempts to estimate a demand curve need to address an endogeneity issue.

We may think that this problem can be addressed with instrumental variables regression. In practice we are usually not interested in the aggregate demand curve but rather the demand curve for many (differentiated) products. This means we would need to estimate estimate a large number of parameters because we would have a demand curve for each differentiated good; a problem that becomes even harder if there are substitution patterns. Furthermore, consumers' tastes are inherently heterogenous and thus assuming a representative individual may be unwarranted.

The problem of dimensionality can be solved by projecting products into a space of characteristics, and use a random utility model to approximate consumers decisions in a way that is theoretically consistent, controlling for observables. The problem is that substitution between products is driven by market shares and not by how similar the products are (i.e., elasticities of demand). Moreover, estimation treats regressors as exogenously estimated.

A potential way to address these issues are Recursive Mixed-Process Models, however these models implictly assume that if the price of a given product rises, it would lose its customers equally to each other product. This assumption is unwarranted. Even if there is a possibility of using a nested structure, a prior must be established to construct the nesting, but this would imply strong assumptions about the structure of the nesting and its representativeness.

Structural methods allow to address the aforementioned issues by imposing few assumptions in a theoretically consistent model of demand. First of all, the structural makes us think about the problem of endogeneity seriously because theory describes with precision where would these problems arise from, conditional on data availability: A product is defined by a set of characteristics. Producers and consumers observe all product characteristics. Each product will be assumed to have a characteristic that influences demand that either is unobservable or unquantifiable.

Second, it allows us to recognize that the problem can be complex enough that imposing linearity assumptions to a model of demand is unwarranted. The structural approach allow us to separate the linear and nonlinear estimation problems. The estimation is divided into one part that uses a search algorithm to numerically estimate parameters that enter nonlinearly into consumers' utility function and a second part that uses an analytic formula to estimate the parameters that enter linearly.

Finally, the contribution of the estructural estimation method is to transform the model in such a way that instrumental-variable methods can be used to address the endogeneity issue aforementioned.

## The model

Consider $t = 1,...,T$ markets each with $i=1,...,I$ consumers and $J$ products, for which we observe quantities, prices, and characteristics. Define the indirect utility function of consumer $i$ from consuming product $j$ in market $t$ as follows:

\begin{equation}
    u_{ijt}=\alpha_i(y_i-p_{jt})+x_{jt}\beta_i+\xi_{jt}+\epsilon_{ijt},
    \tag{1}
\end{equation}

where
- $y_i$ denotes $i$'s income,
- $p_{jt}$ is price of product $j$ in market $t$,
- $x_{jt}$ us $K$-dimensional vector of characteristics for product $j$,
- $\xi_{jt}$ is an unobserved product characteristics,
- $\epsilon_{ijt}$ is mean zero stochastic term,
- $\alpha_i$ is $i$'s marginal utility from income,
- $\beta_i$ is individual-specific taste.

Note this function is quasilinear in income, not concave, hence we can ignore individuals' incomes. This is the main difference with BLP (1995). Thus we denote income as $y_i$, constant across markets, rahter $y_{it}$, varying across them. But if the income effect is important, it can be modelled by using an indirect utility that is a non-linear function of $y_{it}-p_{jt}$, as in BLP (1995).

### Taste parameters

Consumer preferences vary as a function of the individual characteristics. In the context of Equation (1) this amounts to modeling the distribution of consumer taste parameters. We assume that the taste parameters vary with observed $D_i$ and unobservable $\nu_i$ features:

$$\left(\begin{array}{c} \alpha_i \\ \beta_i \end{array} \right) = \left(\begin{array}{c} \alpha \\ \beta \end{array} \right) + \left(\begin{array}{c} \Pi_\alpha \\ \Pi_\beta \end{array} \right) D_i +\left(\begin{array}{c} \Sigma_\alpha \\ \Sigma_\beta \end{array} \right) (\nu_{i\alpha},\nu_{i\beta}), \hspace{1em} \nu_i \sim P_{\nu}^*(\nu), \hspace{1em} D_i \sim \widehat{P}^*_D(D),$$

where $D_i$ is a $d \times 1$ vector of demographic characteristics and $v_i$ is a vector of additional (unobservable) characteristics; $\Pi$ is a $(K+1)\times d$ matrix of parameters that measure how the taste characteristics vary with demographics; $\Sigma$ is a $(K+1)\times (K+1)$ that measure the effect of unobservables on taste characteristics. (Note that I split each vector or matrix into two parts.) Finally, $\nu_i$ and $D_i$ are assumed to be independent and $P_{\nu}^*(\nu)$ is multivariate normal. Thus even though we do not observe individual data, we know something about the distribution of the demographics, while for the additional characteristics we have no such information.

Replacing the previous expression in the utility function yields

$$u_{ijt} = \alpha_iy_i -(\alpha+\Pi_\alpha D_i+\Sigma_\alpha\nu_{i\alpha})p_{jt}+x_{ht}(\beta+\Pi_\beta D_i+\Sigma_\beta\nu_{i\beta})+\xi_{jt}+\epsilon_{ijt}.$$

After re-arranging we find the following expression

$$u_{ijt} = \alpha_iy_i+ \underbrace{(x_{jt}\beta + \alpha p_{jt} + \xi_{jt})}_{\delta_{jt}} + \underbrace{[-p_{jt}, x_{jt}](\Pi D_i + \Sigma \nu_i)}_{\mu_{ijt}} + \epsilon_{ijt}, $$
where $\delta_{jt}$ are fixed effects that do not vary by consumer (referred to as mean utility by many) and $\mu_{ijt}$ are random effects. Let $\theta = (\theta_1, \theta_2)=(\theta_1, \theta_2)$, where $\theta_1=(\alpha,\beta)$ contains the linear parameters of the model and $\theta_2=(\Pi,\Sigma)$ the non-linear ones.

The problem of selection on the dependent variable is solved by assuming a sythentic good (or baseline brand): the consumer may not purchase any of the brands, and thus we normalize the utility that consumers derive from it to zero ($u_{i0t} = 0$).

### Demand for products

Consumers are assumed to purchase one unit of the good that gives the highest utility. The section of parameter space which defines the individuals who prefer to buy good $j$ to good $k$ in market $t$ is

$$A_{jt}(x_{.t}, p_{.t}, \delta_{.t}; \theta_2) = \{ (D_i, \nu_i, \epsilon_{i0t}, ..., \epsilon_{iJt})|u_{ijt} > u_{ikt}\}.$$

Assuming ties occur with zero probability, the market share of the jth product is just an integral over the mass of consumers in the region $A_{jt}$:

\begin{align}
s_{jt}(x_{.t}, p_{.t}, \delta_{.t}; \theta_2) &= \int_{A_{jt}} dP^*(D, \nu, \epsilon) \\
&= \int_{A_{jt}} dP^*(\epsilon|D, \nu) dP^*(\nu|D) dP^*_D(D) \\
&= \int_{A_{jt}} dP^*_{\epsilon} dP^*_\nu(\nu) d\hat{P}^*_D(D)
\end{align}

which a consequence of the independence assumptions previously made.

### Distributional assumptions

The assumption that errors are independent implies that consumers substitute towards other brands in proportion to market
shares, regardless of characteristics. Thus we cannot recover sensible estimates of cross-price elasticities. To induce a correlation among options we can allow the error term to be correlated across products rather than independently distributed. This is done by imposing an i.i.d. extreme-value distribution assumption on $\epsilon_{ijt}$ (i.e., type I extreme value or Gumbel), and allowing the correlation between choices is through $\mu_{ijt}$ (McFadden, 1978). In other words, assume that the observable part is specified sufficiently that the remaining, unobserved portion of utility is essentially white noise.

Using this assumption the probability that $i$ choses $j$ is

\begin{align}
Pr(\alpha_i y_i + \delta_{jt} + \mu_{ijt} + \epsilon_{ijt} > \alpha_i y_i + \delta_{kt} + \mu_{ikt} + \epsilon_{ikt})&=Pr( \delta_{jt} -  \delta_{kt}  + \mu_{ijt}-\mu_{ikt} + \epsilon_{ijt} >   \epsilon_{ikt})\\
&=Pr( z_{ijkt} >   \epsilon_{ikt})\\
&=P^*_\epsilon(z_{ijkt})
\end{align}
where $z_{ijkt}\equiv \delta_{jt} -  \delta_{kt}  + \mu_{ijt}-\mu_{ikt} + \epsilon_{ijt}$.

Assuming that $\epsilon_{ijt}$ is distributed type-I extreme value, we have
\begin{equation}
P^*_\epsilon(z_{ijkt}) = exp(-exp(-z_{ijkt}))
\end{equation}

is the cumulative distribution for each $\epsilon_{ikt}$ evaluated at $z_{ijkt}$. Since the errors are assummed independent, this cumulative distribution is therefore the product of the individual cumulative distributions:
$$\prod_{k\neq j} P^*_\epsilon(z_{ijkt}) = \prod_{k\neq j} exp(-exp(-z_{ijkt}))$$

Since $\epsilon_{ijt}$ is not given, the choice probability is the integral over all values of $\epsilon_{ikt}$  weighted by its density. Thus the market share of product $j$ in market $t$ in $i$'s consumption is
$$s_{ijt}=\int \left( \prod_{k\neq j}  e^{-e^{z_{ijkt}}}\right)e^{-\epsilon_{ijt}}e^{-e^{\epsilon_{ijt}}} d \epsilon_{ijt}$$
which after some derivations provides us with the following expression:
$$s_{ijt}=\frac{e^{\delta_{jt}+\mu_{ijt}}}{1+\sum_k e^{\delta_{kt}+\mu_{ikt}}}$$

Then let us use this expression to compute the market share of product $j$ in market $t$ by integrating the market shares across consumers, which is just an integral over the mass of consumers in region $jt$:

\begin{equation}
    s_{jt}=\int_D\int_\nu s_{ijt} dP_{\nu}^*(\nu)d \hat{P}_D^*(D),
    \tag{2}
\end{equation}

The price elasticity of the market share of product j with respect to the price of $k$ is

$$\eta_{jkt}=\frac{\partial s_{jt}}{\partial p_{kt}}\cdot\frac{p_{kt}}{s_{jt}}=\begin{cases}
-\frac{p_{jt}}{s_{jt}}\int_D\int_\nu \alpha_i s_{ijt}(1-s_{ijt}) dP_{\nu}^*(\nu)d \hat{P}_D^*(D) & if\; j=k,\\
\frac{p_{kt}}{s_{jt}} \int_D\int_\nu \alpha_i s_{ijt}s_{ikt} dP_{\nu}^*(\nu)d \hat{P}_D^*(D) & otherwise.
\end{cases}$$

The full model also allows for flexible substitution patterns, which are not constrained by a priori segmentation of the market (yet at the same time can take advantage of this segmentation by including a segment dummy variable as a product characteristic). This does not come without a cost:

$$u_{ijt} = \alpha_iy_i+\delta_{jt} +\mu_{ijt} + \epsilon_{ijt}$$

does not have an analytic closed form. Furthermore, the computation of the integral in $s_{jt}$ is difficult. This problem
is solved using numerical methods.

## Estimation

Replication data from: http://www.rasmusen.org/zg604/lectures/blp/frontpage.htm

Or: http://faculty.wcas.northwestern.edu/~ane686/supplements/rc_dc_code.htm

There are essentially four steps (plus an initial step) to follow in computing the estimates:
0. prepare the data including draws from the distribution of v and D for $i=1,...,ns$ for a sample of size $ns$, and use the same draws for each so that the predicted market shares sum up to one;
1. for a given (arbitrary) value of $\theta_2$ and $\delta$, compute the market shares;
2. for a given  $\theta_2$, compute the vector $\delta$ that equates the market shares computed in Step 1 to the observed shares;
3. for a given $\theta_2$, compute the error term (as a function of the mean valuation computed in Step 2), interact it with the instruments, and compute the value of the objective function;
. search for the value of $\theta_2$ that minimizes the objective function computed in Step 3.

```python
# First import all the relevant routines
import numpy as np  # Operations with arrays
import pandas as pd # Data base handling package
from scipy.io import loadmat # Scientific computing package - loads matlab matrices
from scipy.optimize import minimize # Scientific computing package - gradient routine
import time # This will help track computing time
```

The estimation solves

$$\min_{\theta}||s(x,p,\delta(x,p,\xi;\theta_1);\theta_2)-S||,$$

where $s(\cdot)$ are the estimated market shares and $S$ are the observed ones. I start by computing the initial values in the numerical iteration.

### Step 0 - prepare the data

```python
#====================================================================================================
# SETTING THE PATHS
#====================================================================================================
main = "C:/Users/cfb310.FAS/"  # for school pc
#main = "D:/Documents/"        # for laptop
path = main+"Dropbox/NYU/Semester VI/Class structural models/Nevo 2000/" # main replication folder
#====================================================================================================
# LOADING DATA

# I use the following convention: T = # of markets; J = # of products; D = # of demograhics
# TJ = T*J = # number of unique market-products; ns = # individuals per market
#====================================================================================================
# ps2.mat contains the matrices v, demogr, x1, x2, s_jt, id_demo
ps2=loadmat(path+'Replication_materials/'+'ps2.mat')

# We convert each element into an array
v       =ps2['v']       # (T by ns*D) = (94 by 20*4) - Unobserved characteristic
demogr  =ps2['demogr']  # (T by ns*D) = (94 by 20*4) - Sociodemographics
x1      =ps2['x1']      # (T*J by k1) = (2256,25)    - Product attributes that enter the linear part
x2      =ps2['x2']      # (T*J by k2) = (2256,4)     - Product attributes that enter the non-linear part
s_jt    =ps2['s_jt']    # (T*J by 1) = (2256,1)      - Observed market shares
id      =ps2['id']      # T*J = 2256                 - id for brand-city-year-quarter
id_demo =ps2['id_demo'] # T = 94                     - id variable for random draws and demographics
#====================================================================================================
# iv.mat contains the variable iv which consists of an id column (see the id variable above) and 20
# columns of IV's for the price variable. The variable is sorted in the same order as variables in ps2.mat
iv =loadmat(path+'Replication_materials/'+'iv.mat')
```

`id` - an id variable in the format bbbbccyyq, where bbbb is a unique 4 digit identifier for each brand  (the first digit is company and last 3 are brand, i.e., 1006 is K Raisin Bran and 3006 is Post Raisin  Bran), cc is a city code, yy is year (=88 for all observations is this data set) and q is quarter. All  the other variables are sorted by date city brand.

`id_demo` - an id variable for the random draws and the demographic variables, of the format ccyyq. Since these variables do not vary by brand they are not repeated. The first observation here corresponds to the first market, the second to the next 24 and so forth.

`s_jt` - the market shares of brand j in market t. Each row corresponds to the equivalent row in id.

`x1` - the variables that enter the linear part of the estimation. Here this consists of a price variable (first column) and 24 brand dummy variables. Each row corresponds to the equivalent row in id. This matrix is saved as a sparse matrix.

`x2` - the variables that enter the non-linear part of the estimation. Here this consists of a constant, price, sugar content and a mushy dummy, respectively . Each row corresponds to the equivalent row in id.

`v` - random draws given for the estimation. For each market 80 iid normal draws are provided. They correspond to 20 "individuals", where for each individual there is a different draw for each column of x2. The ordering is given by id_demo.

`demogr` - draws of demographic variables from the CPS for 20 individuals in each market. The first 20 columns give the income, the next 20 columns the income squared, columns 41 through 60 are age and 61 through 80 are a child dummy variable (=1 if age <= 16). Each of the variables has been DEMEANED (i.e. the mean of each set of 20 columns over the 94 rows is 0). The ordering is given by id_demo.

```python
# The matrix of instruments has 20 instruments for price and the 24 brand dummy variables
IV = np.matrix(np.concatenate((iv['iv'][:, 1:], x1[:, 1:].todense()), 1))
#====================================================================================================
# Defining the dimensions of the data available
K1 = x1.shape[1]        # price var + 24 brand dummy vars
K2 = x2.shape[1]        # 4 product characteristics
ns = int(v.shape[1]/K2) # 20 individuals in each market
T = v.shape[0]          # 94 markets
J = int(x2.shape[0]/T)  # 24 products/brands
TJ = x1.shape[0]        # (94 markets * 24 of products)
#====================================================================================================
#  T*J size vector of market indexes
cdid = np.repeat(np.arange(0,T),J)

#  T size vector with the index of the last observation for a market
cdindex = np.arange(J-1,J*(T),J)

# Note: it is important to recall that in Python the first element is position zero. I correct for this.
#====================================================================================================
# Expand v and demogr to create an observation per market-product
vfull = v[cdid]
dfull = demogr[cdid]

# Number of demographic variables
D  = int(dfull.shape[1]/ns) # 4 observables
```

### Step 1 - compute inital market shares

Recall that the individual probabilities of choosing each brand: $s_{ijt}=\frac{e^{\delta_{jt}+\mu_{ijt}}}{1+\sum_k e^{\delta_{kt}+\mu_{ikt}}}$. Thus we can approximate the integral for market share that results from aggregating across $i$with
$s_{jt}=\frac{1}{n}\sum_{j}s_{ijt}$.

```python
#====================================================================================================
# Matrix with starting values for the vector of (non-linear) parameter estimates following pp. 539.
# Columns: income, income^2, age, child
# Rows: Constant, price, sugar, mushy.
#theta2w=  np.array([[0.3302, 5.4819 ,  0     ,  0.2037, 0     ],
#                    [2.4526, 15.8935, -1.2000,  0     , 2.6342],
#                    [0.0163, -0.2506,  0     ,  0.051 , 0     ],
#                    [0.2441, 1.2650 ,  0     , -0.8091, 0     ]])

# To speed up the calculations I use Dari Pus' initial values
theta2w=  np.array([[0.377, 3.089, 0, 1.186, 0     ],
                    [1.848, 16.598, -0.659, 0, 11.625],
                    [0.004, -0.193, 0, 0.029, 0     ],
                    [0.081, 1.468, 0, -1.514, 0     ]])

# There are 13 parameters to be estimated, and 7 set to zero by assumption. There are 4 characteristic
# parameters and 9 interaction  parameters. Parameters set to zero will not be maxed over.
X1_names = ['Price']
X2_names = ['Sugar', 'Mushy']
X3_names = []
D_names = ['Income', 'Income^2', 'Age', 'Child']

# Create a vector of the non-zero elements in the above matrix, and the corresponding row and column
# indices. This facilitates passing values to the functions below.
(theti, thetj)=np.nonzero(theta2w)

# Then we create a vector with all non-zero elements. That is the parameters we will maximize over.
theta2 = theta2w[np.where(theta2w != 0)]

#====================================================================================================
# Obtaining starign value for delta,...

# Create weight matrix for GMM (Hansen, 1982). Errors across markets are assumed to be independent
invA = np.linalg.inv(IV.T @ IV)
#====================================================================================================
# Compute the outside good market share by market (this is a sum-collapse over each market)
temp = np.cumsum(s_jt)
sum1 = temp[cdindex]
sum1[1:] = np.diff(sum1.T)
# We have computed the market share for each market. (Pandas can do this too, but same # of code lines)
outshr = (1.0 - sum1[cdid]).reshape(TJ,1)
# The reshape helps expanding this vector to create an observation per market-product
```

### Step 2 - compute initial value of $\delta$ that equates market shares

To retrieve $\delta_{jt}$ we need to compute the structural error term $\xi_{jt}$. We need to express the error term as an explicit linear function of the parameters of the model and the data. In order to do this, we solve for each market the implicit system of equations
$$s(\delta_t;\theta_2)=S_{.t}$$
where $s(\cdot)$ are the predicted market shares obtained from Equation (2) and $S$ are the observed market shares.

Using the computation of the market share, we invert the system of equations. This inversion can be computed analytically by

$$\delta_{jt} = ln(S_{jt}) - ln(S_{0t})$$

that is by taking the log of the ratio between the observed market share of product $j$ in $t$ and the observed share of the external good in market $t$.

A good starting value for $\delta_{jt}^{0}=\ln(S_{jt})-\ln(S_{0t})$, where $S_{0t}$ denotes the outside good.

```python
#====================================================================================================
# Initialize mean utility: logit results
y = np.log(s_jt) - np.log(outshr)
```

### Step 3 - compute the error term and the objective function

Once the inversion has been computed, the error term is defined as

$$\omega_{jt}=\delta_{jt}(S_{.t};\theta_2)-\underbrace{(x_{jt}\beta+\alpha_{jt})}_{X_1\theta_1}\equiv \xi_{jt},$$

where $\delta_{jt}(\cdot)$ is the $J\times T$-dimensional vector computed in Step 2 and $X_1$ is the design matrix defined in step 0. Note that the brand-specific dummy variables in $X_1$ capture the characteristics that do not vary by market and the product-specific mean of unobserved components, namely, $x_{jt}\beta+\xi_{jt}$.

Let $Z=[z_1,...,z_m]$ be a set of isntruments such that $E[Z_m\omega(\theta^\star)]=0$, where $\omega(\cdot)$ is a function of the model parameters and $\theta^\star$ denotes the true values of the parameters. The problem is then to choose $\theta$ such that minimizes $M'\Phi^{-1}M$ where $M=Z\omega(\theta)$ and $\Phi$ is the variance-covariance matrix of the moment conditions.

The GMM estimate is
$$\hat{\theta}\equiv \underset{\theta}{argmin}\; \omega(\theta)'Z\Phi^{-1}Z'\omega(\theta),$$

where where $\Phi$ is a consistent estimate of $E[Z'\omega\omega'Z]$. Since the errors are homoscedastic the optimal weight matrix is proportional to $Z'Z$, thus, the need to know the true parameter values is eliminated.

We obtain

$$\widehat{\theta_1}^0=(X_1'Z\Phi^{-1}Z'X_1) X_1'Z\Phi^{-1} \delta_{jt}^{0}.$$

```python
#====================================================================================================
# Use the IV moment conditions E(Z'\Xi)=0 and define the GMM estimator using
mid = x1.T @ IV @ invA @ IV.T
t = np.linalg.inv(mid @ x1) @ mid @ y
d_old = x1 @ t
# initial value of theta_1, note that y is the same as the initial value of delta such that market shares
# are equal to observed market shares. d_old the vector of brand-dummy coefficients.
```

### Iterate 1-3 for the full random-coefficients

For the full random-coefficients model the system of equations is nonlinear and is solved numerically. It can be solved by using the following contraction mapping:

$$\delta_{.t}^{h+1}=\delta_{.t}^{h+1}+\ln S.t - \ln s.{t}(p_{.t},x_{.t},\delta_{.t}^h,P_ns;\theta_2),$$

whre $s(\cdot)$ are the predicted market shares, $h\in \{1,...,H\}$ is the smallest integer shuch that $||\delta_{.t}^H-\delta_{.t}^{H-1}||$.

Since exponents and logarithms is computationally demanding for the contraction mapping, solve for the exponent of the vector
$\delta$, that is $w_{jt}=exp(\delta_{jt})$, then

$$w_{jt}^{h+1}=w_{jt}^h\frac{S_jt}{s_{jt}(x_{.t},p_{.t},\delta^h_{.t};\theta_2)}$$

thus for the initial value $w_{t}^0=exp(\delta^0_{.t})$.

Then we search for the value of $\theta$ that minimizes the objective function. For the full model we need to perform a non-linear search over $\theta_2$. The time required for this search can be reduced by using the first order conditions, with respect to $\theta_1$ as a function of $\theta_2$:

$$\widehat{\theta_1}=(X_1'Z\Phi^{-1}Z'X_1) X_1'Z\Phi^{-1} \delta_{jt}(\widehat{\theta}_2).$$

```python
#====================================================================================================
# save the mean utility as initial values for the search below.
d_old = np.exp(d_old)
gmmvalold = 0
gmmdiff = 1
iter = 0
#====================================================================================================
# NOTE: BEFORE RUNNING THIS LINE MAKE SURE OF RUNNING ALL FUNCTIONS DEFINED BELOW, IN THE APPENDIX

starttime = time.time()

# Running the optimization routine an replacing theta
theta2 = iterate_optimization(opt_func=gmmobj, param_vec=theta2,args=())
# Thus for given values of the nonlinear parameters theta_2, we solve for the mean utility levels delta
# that set the predicted market shares equal to the observed market shares.

print('run time: {} (minutes)'.format((time.time() - starttime) / 60))

# Average time to completion:
```

    run time: 4.696386261781057 (minutes)

A serious objection to the use of brand dummy variables is that the taste coefficients $\beta$ cannot be identified. Fortunately, this is not true. The taste parameters can be retrieved by using a minimumdistance
procedure (as in Chamberlain, 1982). Let $d = (d_1,...,d_j)$ denote the $J \times 1$ vector of brand dummy coefficients, $X$ be the $J × K(K < J)$ matrix of product characteristics that are fixed across markets, and $\xi = (\xi,...,\xi_j)'$ be the $J \times 1$ vector of unobserved product qualities. Then

$$d=X\beta+\xi$$

If we assume that $E[\xi|X] = 0$ we obtain for then the estimates of $\beta$ and $\xi$ are

$$\widehat{\beta}=(X'V_d^{-1}Z'X) X'V_d^{-1} \widehat{d},\quad \widehat{\xi}=\widehat{d}-X\widehat{\beta}.$$

where $\widehat{d}$ is the vector of coefficients estimated from the procedure described previously, and $V_d$ is the variance-covariance matrix of these estimates. The minimum-distance procedure project these estimates onto a lower-dimensional space, which is implied by a restricted model that sets $\xi$ to zero.

```python
#====================================================================================================
# Estimate variance-covariance matrix for minimum distance procedure
var = varcov(theta2)
se_all = np.sqrt(var.diagonal())

# The Minimum Distance estimates if brand dummies were included, they absorb any product
# characteristics that are constant across markets.
if ((K1-len(X1_names+X3_names))>0) & (len(X2_names)>0):
    omega = np.linalg.inv(var[K1-J:K1,K1-J:K1])
    xmd = x2[0:J,[0]+list(range(K2-len(X2_names),K2))]
    ymd = theta1[K1-J:K1]

    beta = np.linalg.inv(xmd.T @ omega @ xmd) @ xmd.T @ omega @ ymd
    resmd = ymd - xmd @ beta;
    semd = np.sqrt((np.linalg.inv(xmd.T @ omega @ xmd)).diagonal())

    Rsq = 1-((resmd-np.mean(resmd)).T @ (resmd-np.mean(resmd))) @ np.linalg.inv((ymd-np.mean(ymd)).T @ (ymd-np.mean(ymd)))
    Rsq_G = 1-(resmd.T @ omega @ resmd) @ np.linalg.inv((ymd-np.mean(ymd)).T @ omega @ (ymd-np.mean(ymd)))
else:
    beta = []
    semd = []
    Rsq = []
    Rsq_G = []

#mcoef = np.concatenate((beta[0,0], theta1[0,0:X1.shape[1]], beta[0,1:], theta1[0,X1.shape[1]:K1-J]),0)
#semcoef = np.concatenate((semd[0,0], se_all[0,0:X1.shape[1]], semd[0,1:], se_all[0,X1.shape[1]:K1-J]),0)

mcoef = np.concatenate((beta[0,:], theta1[0,0:len(X1_names)], beta[1:,:]),0)
semcoef = np.concatenate((semd[:,0], se_all[0,0:len(X1_names)], semd[:,1:].T),0)

index = []
names = ['Constant'] + X1_names + X2_names + X3_names
for var in names:
    index.append(var)
    index.append('')

table_results = pd.DataFrame(data=np.zeros((K2 * 2, 2 + D)), columns=['Mean', 'SD'] + D_names)
se_theta2 = se_all[:,K1:]
se_theta2w = np.zeros((1,K2*(1 + D)))
for i in range(se_theta2.shape[1]):
    se_theta2w[0,rel[i]] = se_theta2[0,i]
se_theta2w = se_theta2w.reshape((K2,1+D), order='F')
for i in range(len(names)):
   table_results.loc[2*i+1, 'Mean'] = semcoef[i,0]
   table_results.loc[2*i+1, ['SD'] + D_names] = se_theta2w[i,:]

table_results.index=index
for i in range(len(names)):
    table_results.loc[names[i],'Mean']= mcoef[i]
theta2w = np.zeros((K2, D+1))
for ind in range(len(theti)):
    theta2w[theti[ind], thetj[ind]] = theta2[ind]
table_results.loc[names,['SD'] + D_names]=theta2w

table_results = table_results.reset_index()
```

Finally we present the results

```python
print(table_results)
print('GMM objective: {}'.format(gmmvalnew))
print('Min-Dist R-squared: {}'.format(Rsq[0,0]))
print('Min-Dist weighted R-squared: {}'.format(Rsq_G[0,0]))
```

          index       Mean        SD      Income   Income^2       Age      Child
    0  Constant  -2.010198  0.558228    2.292376   0.000000  1.284481   0.000000
    1             0.327063  0.162589    1.209040   0.000000  0.631173   0.000000
    2     Price -62.748438  3.313584  588.675608 -30.210302  0.000000  11.053792
    3            14.810502  1.340933  270.580655  14.108635  0.000000   4.122543
    4     Sugar   0.116253 -0.005793   -0.385094   0.000000  0.052249   0.000000
    5             0.016040  0.013507    0.121525   0.000000  0.025993   0.000000
    6     Mushy   0.499535  0.093504    0.747860   0.000000 -1.353348   0.000000
    7             0.198620  0.185438    0.802442   0.000000  0.667079   0.000000
    GMM objective: 4.561516417265565
    Min-Dist R-squared: 0.45910452799355916
    Min-Dist weighted R-squared: 0.10118278719742801

(*The intepretation herein is borrowed almost word-by-word from Nevo*)

The means of the distribution of marginal utilities ($\beta$) are presented in the first column. Therein the mean price coefficient is negative and utility increases with the sugar content.

Estimates of heterogeneity around these means are presented in the next few columns. The column labeled ''Standard Deviations'' captures the effects of the unobserved demographics. All the estimates of the standard deviations are economically insignificant, suggesting that the heterogeneity in the coefficients is mostly explained by the included demographics. These results do not suggest that observed demographics can explain all heterogeneity; they only suggest that the data rejects the assumed normal distribution.

The last four columns present the effect of demographics on the slope parameters. The point estimates are economically significant. Coefficients on the interaction of price with demographics are economically significant, while the estimate of the standard deviation suggests that most of the heterogeneity is explained by demographics. Children and above-average-income consumers tend to be less price-sensitive. The estimates also suggest that while the average consumermight like a soggy cereal, the marginal valuation of sogginess decreases with age and income. In other words, adults are less sensitive to the crispness of a cereal, as are the wealthier consumers.

Most of the coefficients are not statistically significant. This is because the data herein is limited. If the focus were on the real use of these estimates, their efficiency could be greatly improved by using more data, increasing the number of simulations ($ns$), improving the simulation methods, or adding a supply side (this is outside of the scope of this replication exercise).

## References

<a id='nevo_2000'></a>
Nevo, A. (2000). A practitioner's guide to estimation of random‐coefficients logit models of demand. Journal of economics & management strategy, 9(4), 513-548.

Berry, S., Levinsohn, J., & Pakes, A. (1995). Automobile prices in market equilibrium. Econometrica: Journal of the Econometric Society, 841-890.

McFadden, D. (1978). Modeling the choice of residential location. Transportation Research Record, (673).

Chamberlain, G. (1980). Multivariate Refression Models for Paned Data (No. t0008). National Bureau of Economic Research.

# Appendix

## Running the batch of defined functions

Run all the functions defined below before executing the optimization process.

### mufunc

```python
def mufunc(theta2w):
    # args:
    #   theta2 is (Pi, Sigma)
    #  (not providing other stuff as args here - they remain constant throughout the problem)
    '''This function computes the non-linear part of the utility (mu_ijt)'''

    # create a two-dimensional array of zeroes
    mu = np.zeros((TJ,ns))
    # iterate through individuals i (1 thru 20)
    for i in range(ns):
        # v_i is (2256 x 4): 4 unobservables for indiv i across all markets
        v_i = vfull[:, np.arange(i, K2*ns, ns)]
        # d_i is (2256 x 4): 4 demographics for indiv i across all markets
        d_i = dfull[:, np.arange(i, D*ns, ns)]
        # temp will be: 94 x 4
        temp = d_i @ theta2w[:, 1:(D+1)].T  # (Take constant out)
        # np.multiply is elementwise product
        mu[:, i]=(np.multiply(x2, v_i) @ theta2w[:, 0]) + np.multiply(x2, temp) @ np.ones((K2))
        # returns an object of 2256 x 20

    return mu
```

### ind_sh

```python
def ind_sh(theta2w):
    '''This function computes the individual probabilities of choosing each brand'''

    eg = np.multiply(np.exp(mufunc(theta2w)),np.kron(np.ones((1,ns)),d_old))
    temp = np.cumsum(eg,0)
    sum1 = temp[cdindex, :]
    sum2 = sum1
    sum2[1:sum2.shape[0], :] = np.diff(sum1.T).T
    denom1 = 1. / (1. + sum2)
    denom = denom1[cdid, :]

    return np.multiply(eg,denom)
```

### mktsh

```python
def mktsh(theta2w):
    '''This function computes the market share for each product'''

    temp = ind_sh(theta2w).T
    f = (sum(temp)/float(ns)).T
    f = f.reshape(TJ,1)

    return f
```

### meanval

```python
def meanval(theta2):
    '''This function computes the mean utility level'''

    global d_old

    # cutoff values
    if gmmdiff < 1e-6:
        etol = etol = 1e-11
    elif gmmdiff < 1e-3:
        etol = 1e-9
    else:
        etol = 1e-7

    norm = 1
    i = 0
    theta2w = np.zeros((K2, D+1))

    # just copying in the non-zero theta2 values into this theta2w
    for ind in range(len(theti)):
        theta2w[theti[ind], thetj[ind]] = theta2[ind]

    # CONTRACTION MAPPING
    while norm > etol:
        # predict market share from current theta2 values, i.e. compute S(...)
        pred_s_jt = mktsh(theta2w)

        # calculate theta^{h+1} according to the equation above, using the observed market shares s_jt
        # and the theoretical ones just computed S(...)
        #  (actually, eqn 12 with both sides exponentiated)
        d_new = np.multiply(d_old,s_jt) / pred_s_jt

        # t: vector of changes in delta during this iteration
        t = np.abs(d_new - d_old)

        # norm: what was the greatest change in our delta estimates in this iteration
        #   keep iterating until norm is smaller than tolerance level
        # i.e. until we have norm < etol
        norm = np.max(t)
        d_old = d_new
        i += 1

    return np.log(d_new)
```

### gmmobj

```python
# from above:
# IV = np.matrix(np.concatenate((iv['iv'][:, 1:], x1[:, 1:].todense()), 1))

# Recall that IV = [iv, x1] = Z is the matrix of instruments
def gmmobj(theta2):
    '''This function computes the GMM objective function'''

    global gmmdiff; global gmmvalold;  global iter; global gmmresid; global theta1; global gmmvalnew

    # Compute the mean utilities parameters delta performing the contraction mapping
    delta = meanval(theta2)

    # Now from the value of delta obtained we estimate theta1 through GMM (see pag 5 of the Appendix)

    # trick: the following deals with cases were the min algorithm drifts into region where the objective is not defined
    if max(np.isnan(delta)) == 1:
        f = 1e+10
    else:
        # x1: 2256x24, brand dummies
        temp1 = x1.T @ IV

        # compute d'Z
        temp2 = delta.T @ IV

        # compute theta_1 = (x1'Z(Z'Z)^{-1}Z'x1)^{-1} x1'Z(Z'Z)^{-1}Z'd
        theta1 = np.linalg.inv(temp1 @ invA @ temp1.T) @ temp1 @ invA @ temp2.T

        # compute the residual from the GMM estimation (corresponding to the structural error xi)
        gmmresid = delta - x1 @ theta1

        # compute xi'Z = w'Z. This is the vector of moment restrictions that our algorithm has to minimize
        temp1 = gmmresid.T @ IV

        # finally the GMM objective function to be minimized is w'Z(Z'Z)^{-1}Z'w
        f = (temp1 @ invA) @ temp1.T

    # Here I follow Daria
    gmmvalnew = f[0,0]
    gmmdiff = np.abs(gmmvalold - gmmvalnew)
    gmmvalold = gmmvalnew
    iter += 1
    #if iter % 10 == 0:
    #    print('gmm objective:', f[0,0])

    return(f[0, 0])
```

### iterate_optimization

```python
def iterate_optimization(opt_func, param_vec, args=(), method='Nelder-Mead'):
    '''This function computes the gradient of the GMM function'''

    success = False
    while not success:
        res = minimize(opt_func, param_vec, args=args, method=method)
        param_vec = res.x
        if res.success:
            return res.x
```

### jacob

```python
def jacob(theta2):
    '''This function computes the Jacobian of the implicit function that defines the mean utility'''
    global rel

    # Create a matrix of 0s of size (# product characteristics * # observable individual characteristics + 1)
    theta2w = np.zeros((K2, D+1))

    # Re-populate the matrix of parameters with the new inputs of the loop

    # computing (partial share)/(partial sigma)
    for ind in range(len(theti)):
        theta2w[theti[ind], thetj[ind]] = theta2[ind]

    # compute the individual shares for the new values of the parameters
    shares = ind_sh(theta2w)

    # create a matrix of 0s with size ((markets * products) * ((# prod. characteristics * # obs. individ. characteristics + 1))
    f1 = np.zeros((cdid.shape[0] , K2 * (D + 1)))

    # step (1): calculate derivative of shares with respect to the first column of theta2w (variable that are not interacted with demogr var, sigmas)

    # for each product-specific characteristics (i.e. that does not vary by market)
    for i in range(K2):

        # create a matrix of size TJ * ns where the columns relative to the prod. characteristics
        # is repeated ns times (1 for each individual).

        # Then create a matrix of size TJ * ns where the rows (individual characteristics) are repeated for
        # all products/markets and the columns are...
        # Matrix xv should be the matrix of interactions of product characteristics in market t and individual
        # characteristics in market t
        xv = np.multiply(x2[:, i].reshape(TJ, 1) @ np.ones((1, ns)), v[cdid, ns*i:ns * (i + 1)]) # ---> DOUBLE CHECK THIS

        # multiply the matrix above with the individual shares computed above xvs_{ijt}
        # then compute the cumulative sum
        temp = np.cumsum(np.multiply(xv, shares), 0)  # ---> SHARES MUST BE 20*1

        # Keep the last observation for each market
        sum1 = temp[cdindex, :]

        # take the differences to have the shares for each market
        sum1[1:sum1.shape[0], :] = np.diff(sum1.T).T

        # expand the vector of shares to all product/markets, subtract it from the matrix xv, multiply it by the indidividual
        # shares and take the mean. Then flatten the matrix and use it as a column of matrix f1
        f1[:,i] = (np.mean((np.multiply(shares, xv - sum1[cdid,:])),1)).flatten()

    # Step (2): Calculate derivative of shares with respect to all but first column of theta2w
    # (where x2 is interacted with demographic variables)

    # for each demographic variable
    for j in range(D):

        # create a matrix of demographics for each ?individual?
        d = dfull[:, ns * j:(ns * (j+1))]

        # create a matrix of 0s with size TJ * K2
        temp1 = np.zeros((cdid.shape[0], K2))

        # for each product characteristics
        for i in range(K2):
            xd = np.multiply(x2[:, i].reshape(TJ, 1) @ np.ones((1, ns)), d)
            temp = np.cumsum(np.multiply(xd, shares), 0)
            sum1 = temp[cdindex, :]
            sum1[1:sum1.shape[0], :] = np.diff(sum1.T).T
            temp1[:, i] = (np.mean((np.multiply(shares, xd-sum1[cdid, :])), 1)).flatten()
        f1[:, (K2 * (j + 1)):(K2 * (j + 2))] = temp1

    rel = theti + thetj * (max(theti)+1)
    f = np.zeros((cdid.shape[0],rel.shape[0]))
    n = 0

    for i in range(T):
        temp = shares[n:(cdindex[i] + 1), :]
        H1 = temp @ temp.T
        H = (np.diag(np.array(sum(temp.T)).flatten())-H1) / ns
        f[n:(cdindex[i]+1), :] = np.linalg.inv(H) @ f1[n:(cdindex[i] + 1), rel]
        n = cdindex[i] + 1
    return f
```

### varcov

```python
def varcov(theta2):
    '''This function computes the variance-covariance matrix which will be used as weights for the GMM estimator.'''

    # Z is the number of instruments and dummy variables (20 + 24). These are the variables that remove correlation beetween
    # structural error term and mean utility coefficients
    Z = IV.shape[1]

    # compute the jacobian for given theta2
    temp = jacob(theta2)

    # create the matrix a0 = [x1 jacob(theta2) 1] and compute a = a0'Z
    a = np.concatenate((x1.todense(), temp), 1).T @ IV

    # First create a matrix where the GMM residuals are repeated z times (for each instrument), size 2256*44
    # then create a matrix of products of the GMM residuals and the exogenous variables
    IVres = np.multiply(IV , gmmresid @ np.ones((1,Z))) # w(t)

    # compute b as the matrix of cross-products of residuals-instruments
    b = IVres.T @ IVres  # w(t)'w(t)

    # compute intermediate matrix
    inva = np.linalg.inv(a @ invA @ a.T) # (a0'Z (Z'Z)^{-1}Z'a0)^{-1}

    # compute the variance-covariance matrix
    f = inva @ a @ invA @ b @ invA @ a.T @ inva
    # (a0'Z (Z'Z)^{-1}Z'a0)^{-1} a0'Z (Z'Z)^{-1} Z'a0 (a0'Z (Z'Z)^{-1}Z'a0)^{-1}

    return f
```

```python

```


Homework 3

Yutong Dai yutongd3@illinois.edu

## 2.49

### a)
Yes. Consider the following setting:

Denote $A= \begin{bmatrix}1 & -1 \\ 1 & -1\end{bmatrix}$ and $b=(1,-1)^T$. Then the set $\{x|Ax \leq b; x\geq 0\}$ is empty while the set $\{d|Ad\leq 0; d\geq 0, 1^Td=0\}$ has a element of $(0.5, 0.5)^T$.

### b)
There's no relationship between these two. The left figure has a redundant defining plane (1), but has no degeneracy. While the second figure has 1 degree of degeneracy, but has no redundancy.

### c)
Yes. Because in three dimensions or higher, a degenerated extreme point can lie on more than 2 defining planes and remove any of it will change the polyhedron set while it is not true for 2-d space.

### d)
No. Consider $A= \begin{bmatrix}1 & 0 \\ 1 & 0\end{bmatrix}$ and $b=(2, 1)^T$ and the polyhedron set defined by $A$ and $b$, that is $\{x|Ax \leq b\}$. The intersection of two half spaces is $\{(x,y); x\leq 1, y\in R\}$, but there is no extreme point.

### e)
False. Consider a convex cone defined by $n+1$ hyper-planes, then it can have $n+1$ extreme directions. For example, the figure below has more than 3 extreme directions. (Source:Google Image.)

### f)
dim(X) = n - r.

## 2.50

We can have at most $C_6^4=15$ basic solutions. By the code following, we see there are only 13 basic solutions. Out of 13 basic solutions, there are only 4 basic feasible solutions, which implies we have 4 extreme points.

Extreme points are:
```
(0,1,0,1), (0,1,0,0), (0,0,0,1), (0,0,0,0)
```

```python
import numpy as np
A = np.array([[-1,1,-2,0], [-2,0,-1,2], [1,0,0,0], [0,1,0,0], [0,0,1,0], [0,0,0,1]])
b = np.array([[1],[2],[0],[0],[0],[0]])
root = {}
for i in range(0, A.shape[0]-3):
    for j in range(i+1, A.shape[0]-2):
        for k in range(j+1, A.shape[0]-1):
            for l in range(k+1, A.shape[0]):
                try:
                    temp = np.linalg.solve(A[[i,j,k,l], :], b[[i,j,k,l],:])
                    root[str(i)+str(j)+str(k)+str(l)] = temp
                except np.linalg.LinAlgError:
                    pass
```

To find the extreme directions, it's  equivalent to find the basic feasible solutions of the $\{d | Ad\leq 0, d\geq 0\ 1^Td=0\}$.

Extreme directions are:

```
(0, 4/7, 2/7, 1/7)
(1/3, 1/3, 0, 1/3), (0, 0, 2/3, 1/3), (0, 2/3, 1/3, 0)
(0.5, 0, 0, 0.5), (0.5, 0.5, 0, 0)
(1,0,0,0), (0, 1, 0, 0), (0, 0, 1, 0), (0,0,0,1)
```

```python
A = np.array([[-1,1,-2,0], [-2,0,-1,2], [1,0,0,0], [0,1,0,0], [0,0,1,0], [0,0,0,1], [1,1,1,1]])
zero = np.array([[0],[0],[0],[0],[0],[0],[1]])
root = {}
for i in range(0, A.shape[0]-3):
    for j in range(i+1, A.shape[0]-2):
        for k in range(j+1, A.shape[0]-1):
                try:
                    temp = np.linalg.solve(A[[i,j,k,6], :], zero[[i,j,k,6],:])
                    check = True
                    if np.sum([d<0 for d in temp]) > 0:
                        check = False
                    if check:
                        root[str(i)+str(j)+str(k)+str(6)] = [float(x).as_integer_ratio() for x in temp]
                except np.linalg.LinAlgError:
                    pass
```

Choose two extreme points $x_1=(0,1,0,1)$, $x_2=(0,1,0,0)$ and two extreme directions $d_1=(1,0,0,0)$ and $d_2=(0,0,0,1)$. Then
$$0.5x_1 + 0.5x_2 + d_1 + 1.5 d_2 = (1,1,1,2).$$

## Exercise 2.6

a)
If $n\leq m$, then the argument is trivial. Now suppose that $n>m+1$. Pick an arbitrary $y\in C$ and consider the polyhedral set

$$
P = \{\lambda=(\lambda_1,...\lambda_n)| \sum_{i=1}^m \lambda_i A_i=y\}.
$$

By the choice of $y$, we know that there exist an extreme point in P,say $x_0$ (corollary 2.2). So there are at least $n-m$ zero components in $\lambda$, which implies that there are at most $m$ nonzero components in $\lambda$. This concludes the proof.

b)

If $n\leq m$, then the argument is trivial.Now suppose that $n>m+1$. Pick an arbitrary $y\in C$ and consider the polyhedral set

$$
P = \{\lambda=(\lambda_1,...\lambda_n)| \sum_{i=1}^m \lambda_i A_i=y, \sum_{i=1}^n \lambda_i = 1\}.
$$

By the choice of $y$, we know that there exist an extreme point in P,say $x_0$ (corollary 2.2). So there are at least $n-m-1$ zero components in $\lambda$, which implies that there are at most $m+1$ nonzero components in $\lambda$. This concludes the proof.

## Exercise 2.10
a) **TRUE**

proof: WLOG, we assume $A=[A_1, A_2], x=(x_1,x_2)^T$, where $A_1, A_2, x_1, x_2$ are of dimension $m\times m, m\times 1, m\times 1, 1\times 1$ and $A_1$ are of rank $m$. Then we can rewrite $Ax=b$ as $A_1x_1 + A_2x_2=b$. As $A_1$ is invertible, $x_1=A_1^{-1}(b-A_2x_2)$, which implies the dimension of $X$ is actually 1. That is to say, the solutions are always of the form
$$
x= \begin{bmatrix}A_1^{-1}(b-A_2x_2) \\ x_2\end{bmatrix} \overset{\Delta}{=} \begin{bmatrix}a-cx_2 \\ x_2\end{bmatrix}
$$
Note that if we have more than two basic feasible solution, say three, then they must lie on a line segment, which is contained with the $X$. Then the "middle" one can be represent as a strict convex combination of the remaining two, which leads to the contradiction of the definition of the extreme point.

b) **False**

Consider minimizing $0$, subject to $x\geq 0$. The optimal solution set $[0, \infty)$ is unbounded.

c) **False**

Consider the problem
$$
\min_{x\in X} 0,
$$

then all $x\in X$ are optimal, but they are not necessarily to be a basic feasible solution.

d) **True**

The convex combination of any two optimal is another optimal solution.

e) **False**

$$
\begin{align*}
& \min_{x_2} x_2 \\
& s.t. \\
& x_3 = 1 \\
& x_1, x_2, x_3 \geq 0
\end{align*}
$$

Then the optimal solution set is $\{(x_1,0,0)|x_1 \geq 0\}$. But we only have one basic feasible solution set.

f) **False**

Consider the following problem:

$$
\begin{align*}
& \min_{x_1,x_2} \max\{x_1-x_2, x_2-x_1\} \\
& s.t. \\
& x_2 = 1 \\
& x_1 + x_3 = 2\\
& x_1, x_2, x_3 \geq 0
\end{align*}
$$
$(1, 1, 1)$ is the unique optimal solution, but there are only two constraints that are active at $(1, 1, 1)$.

## Exercise 2.12
**True**.

The point $x_0=(0,0,...,0)^T$ is a basic feasible solution. To see this, we notice that there is no such a line that can pass through the $x_0$ and also be contained in the polyhedron set. So $x_0$ is an extreme point.

## Exercise 2.13
### (a)
We proof this by using contradiction.
WLOG, we assume $x=(x_1,...,x_m,0,...,0)^T$, where $x_1,...,x_m >0$, and partition $A$ according to $x$ as $A=[A_M,A_Z]$, where $A_M$ is a $m*m$ matrix.

Suppose $x$ is not a basic feasible solution, then we know that the following $n*n$ matrix's rank is less than  n.
$$
B=
\begin{bmatrix}
A_M & A_Z\\
0 & I_{(n-m)\times (n-m)}
\end{bmatrix}
$$
To simplify the analysis, let's assume that $r(B)=n-1$ and the $m$-th row of $B$ can be write as a non-trivial linear combination of the rest of n-1 rows.
Denote $A_{M-1}$ and $A_{Z-1}$ as the $A_M$ and $A_Z$ without the $m$-th row.

Consider the equation
$$
\begin{bmatrix}
A_{M-1} & A_{Z-1}\\
0 & I_{(n-m+1)\times (n-m+1)}
\end{bmatrix}x=\begin{bmatrix}b \\ 0\end{bmatrix}
$$

In this case, we can have a solution $x^*=(x_1',...,x_{m-1}', 0,...0)$, where  where $x_1',...,x_{m-1}' >0$. Here $x^*$ is a BFS. And the degree of the degeneracy is $1$, as all the rows of the left hand side matrix and the m-th row of $A$ passing through the $x^*$. This contradiction concludes the proof.

### (b)
Consider the following polyhedron set

$$
P= \{(x_1,x_2,x_3|x_2 + x_3 = 0, x_1,x_2,x_3 \geq 0\},
$$

and a point $m_0=(1,0,0)$. Clearly $m_0 \in P$ and is not an extreme point.

## (1)

### Extreme Points.
Some quick observations are,

1. There are 2n+1 constraints. But for n groups of constrains $0\leq w_j \leq \frac{v_j}{v_0}w_0, j=1,...,n$ can not be active at the same time. And it's not possible for $\frac{v_j}{v_0}w_0=0$ without violating the sum-to-one constrain. So any extreme point $(w_0,...,w_n)$, if there is any, for an arbitray choice if $w_j$, can only be active at either $w_j=0$ or $w_j - \frac{v_j}{v_0}w_0=0$.

2. It's impossible for an extreme points to satisfy neither a) $w_j=0,\forall j =1,...,n$ nor b) $w_j -  \frac{v_j}{v_0}w_0=0,\forall j =1,...,n$.

Denote $\mathcal{S}=\{S\subsetneq \{1,2,...,n\} \text{ and } S\neq \emptyset\}$.

So all the extreme points take the form of
$$
(\frac{1}{1+\sum_{j\in S}\frac{v_j}{v_0}}, \frac{1}{1+\sum_{j\in S}\frac{v_j}{v_0}}\frac{v_1}{v_0}I(1\in S), ..., \frac{1}{1+\sum_{j\in S}\frac{v_j}{v_0}}\frac{v_n}{v_0}I(n\in S)), S \in \mathcal{S}
$$

### Equivalence
The proof is a special case of (2) by setting $c$ large enough to make the capacity constraint also hold. Details see the proof given in (2).

## (2)

### Extreme Points.
Similar to (1), there are $n+2$ two constraints. We can choose $m$ constrains from $\{\sum_{i=1}^n w_i=1, \sum_{i=1}^n \frac{w_j}{v_j}=C\frac{w_0}{v_0}\}$ and $n+1-m$ constrains from the pair $(w_j=0, w_j=\frac{v_j}{v_0}w_0)$, $m \in \{1,2\}$. And solve the linear system and check the linear dependency on these $n+1$ constraints. If they are linear independent, we can construct an extreme point, which is quite similar to (1). Otherwise move to another combination of $n+1$ constraints.

If both $\{\sum_{i=1}^n w_i=1, \sum_{i=1}^n \frac{w_j}{v_j}=C\frac{w_0}{v_0}\}$ are active, then the solution is of the form

$$
(\frac{1}{1+\sum_{j\in S}\frac{v_j}{v_0}}, \frac{1}{1+\sum_{j\in S}\frac{v_j}{v_0}}\frac{v_1}{v_0}I(1\in S), ..., \frac{1}{1+\sum_{j\in S}\frac{v_j}{v_0}}\frac{v_n}{v_0}I(n\in S)), S \in \mathcal{S} \text{ and } |S|=C
$$

If only $\{\sum_{i=1}^n w_i=1$ is active, then the solution is the same as the one in (1).

### Equivalence
The original target is to

$$
\begin{align}
& \max_x \frac{\sum_{j=1}^n r_jv_jx_j}{1 + \sum_{j=1}^n v_jx_j}\quad(*) \\
& s.t. \\
& \sum_{j=1}^n x_j\leq c\\
& x_j \in \{0, 1\} \quad j=1,2,...,n
\end{align}
$$

Our goal is to prove the objective function shares the same optimal value as the in one in $(*)$. Without the loss of generality, we can assume that $v_0=1$.

$$
\begin{align}
& \max \sum_{j=1}^nr_jw_j \quad(**)\\
& s.t. \\
& w_0 + \sum_{j=1}^n w_j=1\\
& \sum_{j=1}^n \frac{w_j}{v_j} \leq c w_0\\
& 0  \leq \frac{w_j}{v_j} \leq w_0\quad j=1,2,...,n
\end{align}
$$

Denote $z^*$ be the optimal value of (*) and Consider the problem

$$
\begin{align}
& \max_x \sum_{j=1}^n (r_j-z^*)v_jx_j\quad(***) \\
& s.t. \\
& \sum_{j=1}^n x_j\leq c\\
& x_j \in \{0, 1\} \quad j=1,2,...,n
\end{align}
$$

We assume $(*)$ and $(***)$ achieves the optimal at $x^*$ and $y^*$ respectively. So it is easy to see

$$
z^*(1 + \sum_{j=1}^n v_jx_j^*) = \sum_{j=1}^n r_jv_jx_j^*.
$$

Evaluate the objective function of $(***)$ at $x^*$ (feasible), we have

$$
\sum_{j=1}^n(r_j-z^*)v_jx_j^* = z^*(1 + \sum_{j=1}^n v_jx_j^*) - \sum_{j=1}^n v_jx_j^*z^*=z^*,
$$

which implies the optimal value of $(***)$ is no smaller than $z^*$.

Evaluate the objective function of $(*)$ at $y^*$ (feasible), we have

$$
z^* \geq \sum_{j=1}^n (r_j-z^*)v_jy_j^*
$$

which implies the optimal value of $(*)$ is no smaller than $(***)$.Hence, $(*)$ and $(***)$ have the same optimal value. It remains to be prove $(**)$ and $(***)$ have the same optimal value

Suppose one optimal solution of $(**)$ is $w^*=(w_0^*,...,w_n^*)$ and the optimal value is $a^*$.

Consider a point

$$
\hat w= (\hat w_0,...,\hat w_n),
$$

where $\hat w_j=\frac{v_jy_j^*}{1 + \sum_{j=1}^n v_jy_j^*},\forall 0<j\leq n$ and $\hat w_0 = 1 - \sum_{j=1}^n \hat w_j= \frac{1}{1 + \sum_{j=1}^n v_jy_j^*}$.

It's easy to check that $\hat w$ satisfies all the constraints in $(**)$. So we have

$$
a^* \geq \sum_{j=1}^n r_j\hat w_j = \sum_{j=1}^n r_j\frac{v_jy_j^*}{1 + \sum_{j=1}^n v_jy_j^*}=z^*,
$$

which implies that the optimal value of $(**)$ is no smaller than $z^*$.

Consider a point

$$
\hat y=(\hat y_1, \hat y_2, ...., \hat y_n),
$$

where $\hat y_j = \frac{w_j^*}{w_0^*v_j}$. It's also easy to check that $\hat y$ satisfies all the constraints in $(***)$.
So we have

$$
\begin{align}
z^* & \geq \sum_{j=1}^n (r_j-z^*)v_j\hat y_j\\
& \geq \sum_{j=1}^n (r_j-a^*)v_j\hat y_j = \sum_{j=1}^n (r_j-a^*)\frac{w_j^*}{w_0^*}\\
& = \frac{\sum_{j=1}^n r_j w_j^* - a^*\sum_{j=1}^n w_j^*}{w_0^*} \\
& = \frac{\sum_{j=1}^n r_j w_j^* - a^* (1- w_0^*)}{w_0^*} = a^*
\end{align}
$$

This concludes the proof.

## (3)

We need to to prove $x_iI_{i-1}=0$ holds for all extreme points. We derive this by using contradiction.

For the illustration purpose, we consider i=1,2. Assume $x_2I_1\neq 0$. To to construct an extreme point, one has to satisfy

$$
\begin{bmatrix}
1 & 0 & -1 & 0 \\
0 & 1 & 1 & -1 \\
1 & 0 & 0 & 0 \\
0 & 0 & 0 & 1
\end{bmatrix}
\begin{bmatrix}
x_1 \\
I_1 \\
x_2 \\
I_2
\end{bmatrix}=
\begin{bmatrix}
d_1 \\
d_2 \\
0 \\
0
\end{bmatrix}
$$

It turns out that there is no solution for this linear system, which implies an extreme point must satisfy the ZIO property.

Consider $i=1,2,3$ and the following

$$
A\begin{bmatrix}
 x_{[1:(k+1)]}\\
 I_{[1:(k+1)]}
\end{bmatrix}
=\begin{bmatrix}
 d_{[1:(k+1)]}\\
 0_{[2(k+1)]}
\end{bmatrix}$$

where
$A=\begin{bmatrix}
I_{3} & B\\
C & D
\end{bmatrix}$,
$B_{(3)\times (3)} =
\begin{bmatrix}
-1 & 0 & 0 \\
1 & -1 & 0 \\
0 & 1 & -1\\
\end{bmatrix} \\
$,  and $(C,D) = I_{6\times 6}$.

Note that $x_2I_1=0$, so WLOG, we assume $x_2=0$
If $x_3I_2\neq 0$, then the extreme point has to be active at 2 out of the following 3 constraints.

$$
\begin{cases}
& x_1=  0\\
& I_1 = 0\\
& I_3 = 0
\end{cases}
$$

It turns out that three combinations will lead to infeasible solution. The sample argument applies to $I_1=0$.

Suppose the ZIO property is true for $i=1,2,...,k$. Assume that it's not true that for $k+1$, that is $x_{k+1}I_{k} \neq 0$.

$$A
\begin{bmatrix}
& x_{[1:(k+1)]}\\
& I_{[1:(k+1)]}
\end{bmatrix}
=
\begin{bmatrix}
& d_{[1:(k+1)]}\\
& 0_{[2(k+1)]}
\end{bmatrix}
$$
where

$A=\begin{bmatrix}
I_{k+1} & B\\
C & D
\end{bmatrix}$,
$B_{(k+1)\times (k+1)} =     \begin{bmatrix}
    -1 & 0 & 0 & ... & 0 & 0\\
    1 & -1 & 0 & ... & 0 & 0\\
    0 & 1 & -1 & ... & 0 & 0\\
    \vdots & \vdots & \vdots & ... & \vdots & \vdots \\
    0 & 0  & 0 & ... & 1 & -1\\
    \end{bmatrix} \\
$, and $(C,D) = I_{2(k+1)\times 2(k+1)}$. We can use the same method shown above to see that it will lead to no feasible solution.

So ZIO property is true for $i=1,2,...,12$.

```python

```


```python
# %load /Users/facai/Study/book_notes/preconfig.py
%matplotlib inline

import matplotlib.pyplot as plt
import seaborn as sns
sns.set(style="whitegrid", color_codes=True)
# sns.set(font='SimHei', font_scale=2.5)
plt.rcParams['axes.grid'] = True

import numpy as np
import pandas as pd

np.random.seed(0)

import logging
logging.basicConfig()
logger = logging.getLogger(__name__)

from IPython.display import Image
```

Chapter 2: Multi-armed Bandits
==============

### 2.1 A k-armed Bandit Problem

nonassociative, evaluative feedback problem:

reward = sample(P(action))
+ P(action) is a **stationary** probability distribution.
+ reward: $R$
+ $k$ different actions $a$.
+ goal: maximize the expected total reward over some time period.

The value $q$ of $a$ is the expected reward:

\begin{align}
    q(a) &= \mathbb{E} \left [ R_t \mid A_t = a \right ] \\
         &\approx \frac1{N} \sum_t^N R_t \mid A_t = a \quad \text{law of large numbers} \\
         &= Q_t(a)
\end{align}

### 2.2 Action-value Methods

one natural way to estimate the value of $a$ is by averaging the rewards actually received: (sample-average method)
\begin{equation}
    Q_t(a) = \frac{\sum_{i=1}^{t-1} R_i \cdot \mathbb{1}(A_i = a)}{\sum_{i=1}^{t-1} \cdot \mathbb{1}(A_i = a)}
\end{equation}

strategies:
+ greedy action: $A_t = \operatorname{argmax}_a Q_t(a)$
+ $\epsilon$-greedy actions:

\begin{equation}
    A_t =
    \begin{cases}
        \operatorname{argmax}_a Q_t(a) & \quad \text{ with probability 1 - $\epsilon$} \\
        \text{a random action} & \quad \text{ with probability $\epsilon$}
    \end{cases}
\end{equation}

##### Exercise
2.1:

$0.5 * 1 + 0.5 * 1/2 = 0.75$

2.2:

+ definitely occur: steps = 2, 5
+ possiblly occur: steps= 1, 3, 4

### 2.3 The 10-armed Testbed

```python
reward_funcs = [
    lambda size: np.random.normal(0.2, 1, size),
    lambda size: np.random.normal(-0.8, 1, size),
    lambda size: np.random.normal(1.5, 1, size),
    lambda size: np.random.normal(0.4, 1, size),
    lambda size: np.random.normal(1.2, 1, size),
    lambda size: np.random.normal(-1.3, 1, size),
    lambda size: np.random.normal(-0.2, 1, size),
    lambda size: np.random.normal(-1.0, 1, size),
    lambda size: np.random.normal(1.0, 1, size),
    lambda size: np.random.normal(-0.4, 1, size),
]
```

```python
examples = pd.DataFrame([func(2000) for func in reward_funcs]).T
examples.index.name = "Reward distribution"
plt.figure(figsize=(15, 7))
sns.violinplot(data=examples)
del examples
```

```python
class GreedyAction(object):
    def __init__(self, k, varepsilon=0):
        self.varepsilon = varepsilon
        self.k = k
        # sum of rewards, number of action
        self._records = np.zeros((k, 2))

    def _Q_t(self):
        r = self._records[:, 0]
        n = self._records[:, 1]
        res = np.divide(r, n, out=np.zeros_like(r), where=n!=0)
        logger.info("Q_t: {}".format(res))
        return res

    def action(self):
        if np.random.random() < self.varepsilon:
            a = np.random.randint(low=0, high=self.k)
            logger.info("random action: {}".format(a))
            return a
        else:
            a = np.argmax(self._Q_t())
            logger.info("greedy action: {}".format(a))
            return a

    def update_reward(self, action, reward):
        self._records[action, 0] += reward
        self._records[action, 1] += 1

class Environment(object):
    def __init__(self, reward_funcs):
        self.reward_funcs = reward_funcs
        self.k = len(reward_funcs)

    def feedback(self, action):
        assert 0 <= action < self.k, "action: {} must be in [0, {}]".format(action, self.k)
        return self.reward_funcs[action](1)[0]
```

```python
def run_10_armed(varepsilon):
    opt_action = 2
    env = Environment(reward_funcs)
    actioner = GreedyAction(env.k, varepsilon)
    step_records = []

    def step():
        history = []
        for _ in range(100):
            a = actioner.action()
            r = env.feedback(a)
            actioner.update_reward(a, r)
            logger.info("action: {}, reward: {}".format(a, r))
            history.append([a, r])
        history = np.array(history)
        return np.sum(history[:, 0] == opt_action), np.sum(history[:, 1]), history.shape[0]

    for _ in range(100):
        step_records.append(step())

    result = pd.DataFrame(step_records, columns=["opt_actions", "total_rewards", "iterate_nums"])
    result = result.cumsum()
    result["opt_actions_rate"] = result["opt_actions"] / result["iterate_nums"]
    result["average_rewards"] = result["total_rewards"] / result["iterate_nums"]
    # return result[["opt_actions_rate", "average_rewards"]]
    return result
```

```python
logger.setLevel(logging.WARNING)
```

```python
dfs = [run_10_armed(v) for v in [0.1, 0.01]]
results = pd.concat(dfs, keys=['0.1', '0.01'], axis=1).swaplevel(0, 1, axis=1)
results["average_rewards"].plot()
results["opt_actions_rate"].plot()
```

实验结果和书中有出入

### 2.4 Incremental Implementation

```python
Image('./res/fig_simple.png')
```

General form:

\begin{equation}
    \text{NewEstimate} \leftarrow \text{OldEstimate} + \text{StepSize} \left [ \text{Target} - \text{OldEstimate} \right ]
\end{equation}

### 2.5 Tracking a Nonstationary Problem

it makes sense to give more weight to recent rewards than to long-past rewards: $Q_{n+1} = \alpha R_n + (1 - \alpha) Q_n$ (exponential recency-weighted average)

the conditions required to assure convergence with probability 1:

\begin{align}
    \sum_{n=1}^\infty \alpha_n(a) &= \infty &
    \sum_{n=1}^\infty \alpha_n^2(a) &< \infty &
\end{align}

exercise 2.5:

```python
class NonstationaryEnvironment(object):
    def __init__(self, reward_funcs):
        self.reward_funcs = reward_funcs
        self.k = len(reward_funcs)

        self.iteration = 0

    def feedback(self, action):
        assert 0 <= action < self.k, "action: {} must be in [0, {}]".format(action, self.k)
        self.iteration += 1
        return self.reward_funcs[action](1)[0] + np.random.normal(0, self.iteration / 100, 1)[0]
```

```python
def run_nonstationary_10_armed(varepsilon):
    env = NonstationaryEnvironment(reward_funcs)
    actioner = GreedyAction(env.k, varepsilon)
    step_records = []

    def step():
        history = []
        for _ in range(100):
            a = actioner.action()
            r = env.feedback(a)
            actioner.update_reward(a, r)
            logger.info("action: {}, reward: {}".format(a, r))
            history.append([a, r])
        history = np.array(history)
        return np.sum(history[:, 1]), history.shape[0]

    for _ in range(100):
        step_records.append(step())

    result = pd.DataFrame(step_records, columns=["total_rewards", "iterate_nums"])
    result = result.cumsum()
    result["average_rewards"] = result["total_rewards"] / result["iterate_nums"]
    return result
```

```python
run_nonstationary_10_armed(0.01).plot(x='iterate_nums', y='average_rewards')
```

```python
class ExpGreedyAction(object):
    def __init__(self, k, varepsilon=0, alpha=0.1):
        self.varepsilon = varepsilon
        self.alpha = alpha
        self.k = k
        # sum of rewards, number of action
        self._records = np.zeros((k, 2))

    def _Q_t(self):
        return self._records[:, 0]

    def action(self):
        if np.random.random() < self.varepsilon:
            a = np.random.randint(low=0, high=self.k)
            logger.info("random action: {}".format(a))
            return a
        else:
            a = np.argmax(self._Q_t())
            logger.info("greedy action: {}".format(a))
            return a

    def update_reward(self, action, reward):
        self._records[action, 1] += 1
        self._records[action, 0] += self.alpha * (reward - self._records[action, 0]) / self._records[action, 1]
```

```python
def run_exp_10_armed(varepsilon, alpha):
    env = NonstationaryEnvironment(reward_funcs)
    actioner = ExpGreedyAction(env.k, varepsilon, alpha)
    step_records = []

    def step():
        history = []
        for _ in range(100):
            a = actioner.action()
            r = env.feedback(a)
            actioner.update_reward(a, r)
            logger.info("action: {}, reward: {}".format(a, r))
            history.append([a, r])
        history = np.array(history)
        return np.sum(history[:, 1]), history.shape[0]

    for _ in range(100):
        step_records.append(step())

    result = pd.DataFrame(step_records, columns=["total_rewards", "iterate_nums"])
    result = result.cumsum()
    result["average_rewards"] = result["total_rewards"] / result["iterate_nums"]
    return result
```

```python
run_exp_10_armed(0.1, 0.1).plot(x='iterate_nums', y='average_rewards')
```

### 2.6 Optimistic Initial Values

All the methods we have discussed are biased by their initial estimates.

如2.6式红色部份：

\begin{equation}
    Q_{n+1} = \color{red}{(1 - \alpha)^n Q_1} + \sum_{i=1}^n \alpha (1 - \alpha)^{n-1} R_i
\end{equation}

We call this technique for encouraging exploration optimistic initial values: say, using $Q_1(a) = 5$.

```python
Image('./res/fig2_3.png')
```

Exercise 2.6: 前期有大量探索，所以开始表现不好

Exercise 2.7:

首先归纳：

\begin{align*}
    O_{t+1} &= O_t + \alpha (1 - O_t) \\
            &= \alpha + (1 - \alpha) O_t \\
            &= \alpha + (1 - \alpha) (\alpha + (1 - \alpha) O_{t-1} ) \\
            &= \alpha + (1 - \alpha) \alpha + (1 - \alpha)^2 O_{t-1} \\
            &= \alpha + (1 - \alpha) \alpha + (1 - \alpha)^2 \alpha + \cdots + (1 - \alpha)^{t} O_{t-(t-1)} \\
            &= \alpha + (1 - \alpha) \alpha + (1 - \alpha)^2 \alpha + \cdots + (1 - \alpha)^{t} O_1 \\
            &= \alpha + (1 - \alpha) \alpha + (1 - \alpha)^2 \alpha + \cdots + (1 - \alpha)^{t} \alpha \\
            &= \alpha (1 + (1 - \alpha) + (1 - \alpha)^2 + \cdots + (1 - \alpha)^{t}) \\
            &= \alpha (1 + \sum_{k=1}^{t}(1 - \alpha)^k ) \\
\end{align*}

得到$\beta_n = \frac{\alpha}{O_n} = \frac1{1 + \sum_{t=1}^{n-1} (1 - \alpha)^t}$

且：
\begin{align}
    (1 - \beta_n) \beta_{n-1} &= \beta_n (\frac1{\beta_n} - 1) \beta_{n-1} \\
                              &= \beta_n \frac{\sum_{t=1}^{n-1}}{1 + \sum_{t=1}^{n-2}} \\
                              &= \beta_n \frac{(1 - \alpha) \left (1 + \sum_{t=1}^{n-2} \right )}{1 + \sum_{t=1}^{n-2}} \\
                              &= \beta_n (1 - \alpha)
\end{align}

然后，利用2.5式：

\begin{align*}
    Q_{n+1} &= Q_n + \beta_n (R_n - Q_n) \\
            &= \beta_n R_n + (1 - \beta_n) Q_n \\
            &= \beta_n R_n + (1 - \beta_n) \left ( \beta_{n-1} R_{n-1} + (1 - \beta_{n-1}) Q_{n-1} \right ) \\
            &= \beta_n R_n + \color{blue}{(1 - \beta_n) \beta_{n-1}} R_{n-1} + (1 - \beta_n)(1 - \beta_{n-1}) Q_{n-1} \\
            &= \beta_n R_n + \color{blue}{\beta_n (1 - \alpha)} R_{n-1} + (1 - \beta_n)(1 - \beta_{n-1}) Q_{n-1} \\
            &= \beta_n \left ( R_n + (1 - \alpha) R_{n-1} \right ) + (1 - \beta_n)(1 - \beta_{n-1}) Q_{n-1} \\
            &= \beta_n \left ( R_n + (1 - \alpha) R_{n-1} \right ) + (1 - \beta_n)(1 - \beta_{n-1}) \left ( \beta_{n-2} R_{n-2} + (1 - \beta_{n-2}) Q_{n-2} \right ) \\
            &= \beta_n \left ( R_n + (1 - \alpha) R_{n-1} \right ) + \color{blue}{(1 - \beta_n)(1 - \beta_{n-1}) \beta_{n-2}} R_{n-2} + \prod_{t=0}^2 (1 - \beta_{n-t}) Q_{n-2} \\
            &= \beta_n \left ( R_n + (1 - \alpha) R_{n-1} \right ) + \color{blue}{\beta_n (1 - \alpha)^2} R_{n-2} + \prod_{t=0}^2 (1 - \beta_{n-t}) Q_{n-2} \\
            &= \beta_n \left ( R_n + (1 - \alpha) R_{n-1} + (1 - \alpha)^2 R_{n-2} \right ) + \prod_{t=0}^2 (1 - \beta_{n-t}) Q_{n-2} \\
            &= \beta_n \sum_{t=0}^{2} (1 - \alpha)^t R_{n-t} + \prod_{t=0}^2 (1 - \beta_{n-t}) Q_{n-2} \\
            &= \beta_n \sum_{t=0}^{n-1} (1 - \alpha)^t R_{n-t} + \prod_{t=0}^{n-1} (1 - \beta_{n-t}) Q_{n-(n-1)} \\
            &= \beta_n \sum_{t=0}^{n-1} (1 - \alpha)^t R_{n-t} + \prod_{t=0}^{n-1} (1 - \beta_{n-t}) Q_1 \\
            &= \beta_n \sum_{t=0}^{n-1} (1 - \alpha)^t R_{n-t} + \prod_{t=0}^{n-2} (1 - \beta_{n-t}) \color{red}{(1 - \beta_1)} Q_1 \\
            & \text{因为 $1 - \beta_1 = 1 - 1 = 0$} \\
            &= \beta_n \sum_{t=0}^{n-1} (1 - \alpha)^t R_{n-t} \\
            &= \frac{\sum_{t=0}^{n-1} (1 - \alpha)^t R_{n-t}}{1 + \sum_{t=1}^{n-1} (1 - \alpha)^t} \\
            &= \frac{\sum_{t=0}^{n-1} (1 - \alpha)^t R_{n-t}}{\sum_{t=0}^{n-1} (1 - \alpha)^t} \\
\end{align*}

可以看到，$Q_{n+1}$式中去除了$Q_1$影响，且各项系数累加等于1。故无偏得证。

### 2.7 Upper-Confidence-Bound Action Selection

Select among the non-greedy actions according to their potential for actually being optimal:

\begin{equation}
    A_t \doteq \operatorname{argmax}_a \left [ Q_t(a) + c \sqrt{\frac{\ln t}{N_t(a)}} \right ]
\end{equation}

The square-root term: measure of the uncertainty (or variance) in the estimator of $a$'s value.

```python
Image('res/fig2_4.png')
```

### 2.8 Gradient Bandit Algorithms

Learning a numerical *preference* $H_t(a)$ for each action.

Only the relative preference of one action over another is important, and the action probabilities are determined according to a *soft-max distribution*:

\begin{equation}
    \operatorname{Pr}\{A_t = a\} \doteq \frac{e^{H_t(a)}}{\sum_{b=1}^k e_{H_t(b)}} = \pi_t(a)
\end{equation}

update policy is based on the idea of stochastic gradient ascent:

\begin{align*}
    H_{t+1}(A_t) & \doteq H_t(A_t) + \alpha (R_t - \bar{R}_t) (1 - \pi_t(A_t)) & \text{ and } \\
    H_{t+1}(a) & \doteq H_t(a) - \alpha (R_t - \bar{R}_t) \pi_t(a) & \text{ for all $a \neq A_t$ } \\
\end{align*}

$\bar{R}_t$ is the average of all the rewards up through, which serves as a baseline.

```python
Image('res/fig2_5.png')
```

### 2.9 Associative Search (Contextual Bandits)

+ nonassociative task: stationary task. 没有场景
+ associative task: the goal is to learn a policy: a mapping from situations to the actions that are best in those situations. 有场景的区分
+ full reinforcement learning problem: action are allowed to effec the *next situation* as well as the reward. 行为可以影响场景。

Exercise 2.10:

+ cannot tell which case: 0.5 (random)
+ can tell: A - 2, B - 1: 0.55

### 2.10 Summary

```python
Image('res/fig2_6.png')
```

```python

```