```python
import tensorflow as tf
import tensorflow_probability as tfp
tfd = tfp.distributions

print("TF version:", tf.__version__)
print("TFP version:", tfp.__version__)
```

    TF version: 2.4.1
    TFP version: 0.12.1

```python
# Additional imports and setting fixed random seed to have reproducibility

import matplotlib.pyplot as plt
import numpy as np
tf.random.set_seed(123)
```

# Tensorflow Distributions

## Coding Tutorials
#### 1. [Univariate Distributions](#univariate_distributions)
#### 2. [Multivariate Distributions](#multivariate_distributions)
#### 3. [The Independent Distribution](#the_independent_distribution)
#### 4. [Sampling and log probs](#sampling_and_log_probs)
#### 5. [Trainable Distributions](#trainable_distributions)

***
## Univariate distributions

```python
# Create a normal distribution from Tensorflow Distributions

normal = tfd.Normal(loc=0,scale=1)
normal
```

    <tfp.distributions.Normal 'Normal' batch_shape=[] event_shape=[] dtype=float32>

```python
# Sample from the chosen distribution...

normal.sample(5)
```

    <tf.Tensor: shape=(5,), dtype=float32, numpy=
    array([-0.8980837 , -1.8259144 , -0.44441807, -1.4882947 , -0.7855463 ],
          dtype=float32)>

```python
# ... or sample multiple times

normal.sample(100)
```

    <tf.Tensor: shape=(100,), dtype=float32, numpy=
    array([ 0.33875433,  0.3449861 , -0.6605785 , -0.2854994 ,  0.43852386,
            0.8288566 , -0.53591555, -0.5353483 , -1.0324248 , -2.942705  ,
            0.22994906, -0.54620427, -0.29739588,  0.0165684 ,  0.15928634,
           -0.43964577,  1.2443346 , -0.8386439 , -0.42134959,  0.07302128,
           -0.01232606, -0.45987487, -0.7030544 , -1.0494213 , -0.4068649 ,
            0.70376766,  0.15045081, -0.11866443, -0.97155815, -1.8645742 ,
           -0.9494801 , -1.5251714 , -0.16689245,  0.3013806 ,  2.4803462 ,
           -0.29877564, -0.43377975, -0.7796397 ,  0.12170142,  0.00365537,
            2.3723958 , -0.6633746 , -0.23748903,  0.49617675, -1.2733086 ,
            0.82116693,  1.4974672 ,  1.9248661 ,  1.1244863 , -0.7253876 ,
            1.1786381 ,  2.6358516 ,  1.7394918 , -0.76700944, -0.23252317,
            0.39683342, -0.09006508, -0.26957023, -0.62394035,  1.3852935 ,
            0.03589001, -0.37348393, -2.7601318 , -0.03919969,  1.3688127 ,
           -1.3988788 , -0.80278945, -0.23407489, -1.7803962 , -0.09675159,
           -1.604616  ,  0.78303427,  0.06333537,  0.11498857,  0.8331566 ,
           -1.5389537 , -0.42328802,  0.9145659 ,  0.9061357 , -0.05354716,
            0.60585684, -0.51745254, -1.7043357 , -0.02701364, -0.00710395,
            1.1539234 , -1.7635069 , -2.0365014 , -0.5450172 ,  0.9298993 ,
            0.91419375, -0.3413351 ,  0.5171254 , -0.34438533,  0.5493155 ,
            0.01460932,  0.7466298 ,  0.6907573 ,  0.1272641 ,  0.18322931],
          dtype=float32)>

```python
# Obtain value of probability's density

normal.prob(0)
```

    <tf.Tensor: shape=(), dtype=float32, numpy=0.3989423>

```python
# Obtain value of logprobability

normal.log_prob(0)
```

    <tf.Tensor: shape=(), dtype=float32, numpy=-0.9189385>

```python
# Verify that this really is the log of the probability

np.log(normal.prob(0))
```

    -0.9189385

```python
# Plot a histogram, approximating the density

plt.hist(normal.sample(10000), bins=50, density=True)
plt.show()
```

```python
# Do the same for the exponential distribution

exponential = tfd.Exponential(rate=1)
```

```python
# Sample as before

exponential.sample(100)
plt.hist(exponential.sample(10000), bins=50, density=True)
plt.show()
```

```python
# Create a Bernoulli distribution (discrete)

bernoulli = tfd.Bernoulli(probs = 0.8)
bernoulli.sample(10)
```

    <tf.Tensor: shape=(10,), dtype=int32, numpy=array([1, 0, 1, 1, 1, 1, 1, 1, 1, 1])>

#### A word of caution on discrete distributions

```python
# Calculate Bernoulli prob and see that 0.5 and -1 do not give the correct probability!

for k in [0,0.5,1,-1]:
    print('prob result {} for k = {} '.format(bernoulli.prob(k), k))
```

    prob result 0.19999998807907104 for k = 0
    prob result 0.3999999761581421 for k = 0.5
    prob result 0.800000011920929 for k = 1
    prob result 0.04999999701976776 for k = -1

```python
# Replicate the scores to see what is occurring under the hood

def my_bernoulli(p_success, k):
    return np.power(p_success,k)*np.power((1-p_success),(1-k))
```

```python
# Evaluate it as before

for k in [0,0.5,1,-1]:
    print('prob result {} for k = {} '.format(my_bernoulli(0.8, k), k))
```

    prob result 0.19999999999999996 for k = 0
    prob result 0.3999999999999999 for k = 0.5
    prob result 0.8 for k = 1
    prob result 0.049999999999999975 for k = -1

#### Work with batch distributions

```python
# Create a batched Bernoulli distribution

bernoulli_batch = tfd.Bernoulli(probs=[0.1, 0.25, 0.5, 0.75, 0.9])
bernoulli_batch
```

    <tfp.distributions.Bernoulli 'Bernoulli' batch_shape=[5] event_shape=[] dtype=int32>

```python
# Sample from it, noting the shape

bernoulli_batch.sample(5)
```

    <tf.Tensor: shape=(5, 5), dtype=int32, numpy=
    array([[0, 0, 1, 1, 1],
           [0, 0, 0, 1, 1],
           [0, 0, 1, 1, 1],
           [1, 0, 1, 1, 1],
           [0, 0, 1, 1, 1]])>

```python
# Use a batch shape with higher rank

probs = [[[0.5, 0.5],
          [0.8, 0.3],
          [0.25, 0.75]]]
bernoulli_batch_2D = tfd.Bernoulli(probs=probs)
bernoulli_batch_2D
```

    <tfp.distributions.Bernoulli 'Bernoulli' batch_shape=[1, 3, 2] event_shape=[] dtype=int32>

```python
# Sample from this batch of distributions

bernoulli_batch_2D.sample(5)
```

    <tf.Tensor: shape=(5, 1, 3, 2), dtype=int32, numpy=
    array([[[[0, 1],
             [0, 0],
             [0, 1]]],

           [[[0, 0],
             [0, 0],
             [0, 1]]],

           [[[1, 0],
             [0, 1],
             [1, 0]]],

           [[[0, 0],
             [0, 1],
             [0, 0]]],

           [[[1, 1],
             [0, 0],
             [0, 1]]]])>

```python
# Determine probabilities from this batch distribution

bernoulli_batch_2D.prob([[[1, 0],
                         [0, 0],
                         [1, 1]]])
```

    <tf.Tensor: shape=(1, 3, 2), dtype=float32, numpy=
    array([[[0.5       , 0.5       ],
            [0.19999999, 0.7       ],
            [0.25      , 0.75      ]]], dtype=float32)>

***

## Multivariate Distributions

#### Basic multivariate distributions

```python
# Define 2D multivariate Gaussian with diagonal covariance matrix

normal_diag = tfd.MultivariateNormalDiag(loc = [0,1],
                                        scale_diag = [1,2])

normal_diag
```

    <tfp.distributions.MultivariateNormalDiag 'MultivariateNormalDiag' batch_shape=[] event_shape=[2] dtype=float32>

```python
# Sample from it

normal_diag.sample(10)
```

    <tf.Tensor: shape=(10, 2), dtype=float32, numpy=
    array([[-0.48958328,  2.889566  ],
           [-0.50546694,  0.9930791 ],
           [ 1.300877  ,  1.770391  ],
           [-2.9889584 ,  2.2526274 ],
           [-0.8891398 ,  1.9469274 ],
           [-1.5844265 , -1.2503903 ],
           [-1.0179803 ,  1.302659  ],
           [-0.09586073,  1.3923955 ],
           [-0.3122441 ,  2.9065297 ],
           [ 0.39839435,  0.40829754]], dtype=float32)>

```python
# Make a plot

plt_sample = normal_diag.sample(10000)
plt.scatter(plt_sample[:, 0], plt_sample[:, 1], marker='.', alpha=0.05)
plt.axis('equal')
plt.show()
```

#### Batches of multivariate distributions

```python
# Create three "batches" of multivariate normals

normal_diag_batch = tfd.MultivariateNormalDiag(loc = [[0,0], [0,0], [0,0]],
                                        scale_diag = [[1,2], [2,1], [2,2]])

normal_diag_batch
```

    <tfp.distributions.MultivariateNormalDiag 'MultivariateNormalDiag' batch_shape=[3] event_shape=[2] dtype=float32>

```python
# Sample from it

samples = normal_diag_batch.sample(5)
samples.shape # Sample shape, batch shape, event shape.
```

    TensorShape([5, 3, 2])

```python
# Compute log probs

normal_diag_batch.log_prob(samples)
```

    <tf.Tensor: shape=(5, 3), dtype=float32, numpy=
    array([[-3.5764403, -5.20837  , -5.0379715],
           [-3.8589525, -3.5881023, -9.07949  ],
           [-2.9126272, -3.8017173, -3.2633932],
           [-4.716266 , -2.5851068, -4.162257 ],
           [-3.642995 , -5.021423 , -4.434016 ]], dtype=float32)>

```python
# Create a sample for a plot -- notice the shape

plt_sample_batch = normal_diag_batch.sample(10000)
plt_sample_batch.shape
```

    TensorShape([10000, 3, 2])

```python
# Plot samples from the batched multivariate Gaussian

fig, axs = (plt.subplots(1, 3, sharex=True, sharey=True, figsize=(10, 3)))
titles = ['cov_diag=[1, 2]','cov_diag=[2, 1]', 'cov_diag=[2, 2]']

for i, (ax, title) in enumerate(zip(axs,titles)):
    samples = plt_sample_batch[:,i,:] #take the ith batch [samples x event_shape]
    ax.scatter(samples[:, 0], samples[:, 1], marker='.', alpha=0.05)
    ax.set_title(title)
plt.show()
```

***

## The Independent Distribution

```python
# Start by defining a batch of two univariate Gaussians, then
# combine them into a bivariate Gaussian with independent components

locs = [-1., 1]
scales = [0.5, 1.]

batch_of_normals = tfd.Normal(loc = locs,
                             scale = scales)
```

```python
# Univariate density functions

import seaborn as sns

t = np.linspace(-4, 4, 10000)
densities = batch_of_normals.prob(np.repeat(t[:, np.newaxis], 2, axis=1)) # each column is a vector of densities for one distn

sns.lineplot(t, densities[:, 0], label='loc={}, scale={}'.format(locs[0], scales[0]))
sns.lineplot(t, densities[:, 1], label='loc={}, scale={}'.format(locs[1], scales[1]))
plt.ylabel('Probability density')
plt.xlabel('Value')
plt.legend()
plt.show()
```

```python
# Check their batch_shape and event_shape

batch_of_normals
```

    <tfp.distributions.Normal 'Normal' batch_shape=[2] event_shape=[] dtype=float32>

```python
# Use Independent to convert the batch shape to the event shape

bivariate_normal_from_Independent = tfd.Independent(batch_of_normals,
                                                   reinterpreted_batch_ndims = 1)
```

```python
# Note that dimension from batch_shape has shifted to event_shape

bivariate_normal_from_Independent
```

    <tfp.distributions.Independent 'IndependentNormal' batch_shape=[] event_shape=[2] dtype=float32>

```python
# Create a plot showing joint density contours and marginal density functions

samples = bivariate_normal_from_Independent.sample(10000)
x1 = samples[:, 0]
x2 = samples[:, 1]
sns.jointplot(x1, x2, kind="kde", space=0, color='b', xlim=[-4, 4], ylim=[-4, 4])
```

```python
# Use MultivariateNormalDiag to create the equivalent distribution
# Note that diagonal covariance matrix => no correlation => independence (for the multivariate normal distribution)

bivariate_normal_from_Multivariate = tfd.MultivariateNormalDiag(loc=locs,
                                                               scale_diag = scales)
```

```python
# Plot the joint density function of bivariate_normal_from_Independent
# Refer back to bivariate_normal_from_Independent to show that the plot is the same
# Summarise how Independent has been used

samples = bivariate_normal_from_Multivariate.sample(10000)
x1 = samples[:, 0]
x2 = samples[:, 1]
sns.jointplot(x1, x2, kind="kde", space=0, color='b', xlim=[-4, 4], ylim=[-4, 4])
```

#### Shifting batch dimensions to event dimensions using
`reinterpreted_batch_ndims`

```python
# Demonstrate use of reinterpreted_batch_ndims
# By default all batch dims except the first are transferred to event dims

loc_grid = [[-100., -100.],
            [100., 100.],
            [0., 0.]]
scale_grid = [[1., 10.],
              [1., 10.],
              [1., 1.]]

normals_batch_3by2_event_1 = tfd.Normal(loc=loc_grid, scale=scale_grid)
normals_batch_3by2_event_1
```

    <tfp.distributions.Normal 'Normal' batch_shape=[3, 2] event_shape=[] dtype=float32>

```python
# Highlight batch_shape

normals_batch_3_event_2 = tfd.Independent(normals_batch_3by2_event_1)
normals_batch_3_event_2

# Notice
```

    <tfp.distributions.Independent 'IndependentNormal' batch_shape=[3] event_shape=[2] dtype=float32>

```python
# We now have a batch of 3 bivariate normal distributions,
# each parametrised by a column of our original parameter grid

normals_batch_3_event_2
```

    <tfp.distributions.Independent 'IndependentNormal' batch_shape=[3] event_shape=[2] dtype=float32>

```python
# Evaluate log_prob

normals_batch_3_event_2.log_prob(value = [[-10.,10],
                                            [100.,100.],
                                            [1.,1.]])
```

    <tf.Tensor: shape=(3,), dtype=float32, numpy=array([-4.1146406e+03, -4.1404624e+00, -2.8378770e+00], dtype=float32)>

```python
# Can reinterpret _all_ batch dimensions as event dimensions

normals_batch_1_event_3by2 = tfd.Independent(normals_batch_3by2_event_1,
                                            reinterpreted_batch_ndims = 2)
```

```python
# Take log_probs

normals_batch_1_event_3by2.log_prob(value=[
    [-10.,10.],
    [100.,100.],
    [1.,1.]
])
```

    <tf.Tensor: shape=(), dtype=float32, numpy=-4121.619>

#### Using `Independent` to build a Naive Bayes classifier

#### Introduction to `newsgroups` data set

In this tutorial, just load the dataset, fetch train/test splits, probably choose a subset of the data.

Construct the class conditional feature distribution (with Independent, using the Naive Bayes assumption) and sample from it.

We can just use the ML estimates for parameters, in later tutorials we will learn them.

```python
# Convenience function for retrieving the 20 newsgroups data set

# Usenet was a forerunner to modern internet forums
# Users could post and read articles
# Newsgroup corresponded to a topic
# Example topics in this data set: IBM computer hardware, baseball
# Our objective is to use an article's contents to predict its newsgroup,
# a 20-class classification problem.

# 18000 newsgroups, posts on 20 topics
from sklearn.datasets import fetch_20newsgroups
from sklearn.feature_extraction.text import CountVectorizer

```

```python
# Get the train data

newsgroups_data = fetch_20newsgroups(data_home = '20_Newsgroup_Data/', subset = 'train')
```

```python
# More information about the data set

print(newsgroups_data['DESCR'])
```

    .. _20newsgroups_dataset:

    The 20 newsgroups text dataset
    ------------------------------

    The 20 newsgroups dataset comprises around 18000 newsgroups posts on
    20 topics split in two subsets: one for training (or development)
    and the other one for testing (or for performance evaluation). The split
    between the train and test set is based upon a messages posted before
    and after a specific date.

    This module contains two loaders. The first one,
    :func:`sklearn.datasets.fetch_20newsgroups`,
    returns a list of the raw texts that can be fed to text feature
    extractors such as :class:`sklearn.feature_extraction.text.CountVectorizer`
    with custom parameters so as to extract feature vectors.
    The second one, :func:`sklearn.datasets.fetch_20newsgroups_vectorized`,
    returns ready-to-use features, i.e., it is not necessary to use a feature
    extractor.

    **Data Set Characteristics:**

        =================   ==========
        Classes                     20
        Samples total            18846
        Dimensionality               1
        Features                  text
        =================   ==========

    Usage
    ~~~~~

    The :func:`sklearn.datasets.fetch_20newsgroups` function is a data
    fetching / caching functions that downloads the data archive from
    the original `20 newsgroups website`_, extracts the archive contents
    in the ``~/scikit_learn_data/20news_home`` folder and calls the
    :func:`sklearn.datasets.load_files` on either the training or
    testing set folder, or both of them::

      >>> from sklearn.datasets import fetch_20newsgroups
      >>> newsgroups_train = fetch_20newsgroups(subset='train')

      >>> from pprint import pprint
      >>> pprint(list(newsgroups_train.target_names))
      ['alt.atheism',
       'comp.graphics',
       'comp.os.ms-windows.misc',
       'comp.sys.ibm.pc.hardware',
       'comp.sys.mac.hardware',
       'comp.windows.x',
       'misc.forsale',
       'rec.autos',
       'rec.motorcycles',
       'rec.sport.baseball',
       'rec.sport.hockey',
       'sci.crypt',
       'sci.electronics',
       'sci.med',
       'sci.space',
       'soc.religion.christian',
       'talk.politics.guns',
       'talk.politics.mideast',
       'talk.politics.misc',
       'talk.religion.misc']

    The real data lies in the ``filenames`` and ``target`` attributes. The target
    attribute is the integer index of the category::

      >>> newsgroups_train.filenames.shape
      (11314,)
      >>> newsgroups_train.target.shape
      (11314,)
      >>> newsgroups_train.target[:10]
      array([ 7,  4,  4,  1, 14, 16, 13,  3,  2,  4])

    It is possible to load only a sub-selection of the categories by passing the
    list of the categories to load to the
    :func:`sklearn.datasets.fetch_20newsgroups` function::

      >>> cats = ['alt.atheism', 'sci.space']
      >>> newsgroups_train = fetch_20newsgroups(subset='train', categories=cats)

      >>> list(newsgroups_train.target_names)
      ['alt.atheism', 'sci.space']
      >>> newsgroups_train.filenames.shape
      (1073,)
      >>> newsgroups_train.target.shape
      (1073,)
      >>> newsgroups_train.target[:10]
      array([0, 1, 1, 1, 0, 1, 1, 0, 0, 0])

    Converting text to vectors
    ~~~~~~~~~~~~~~~~~~~~~~~~~~

    In order to feed predictive or clustering models with the text data,
    one first need to turn the text into vectors of numerical values suitable
    for statistical analysis. This can be achieved with the utilities of the
    ``sklearn.feature_extraction.text`` as demonstrated in the following
    example that extract `TF-IDF`_ vectors of unigram tokens
    from a subset of 20news::

      >>> from sklearn.feature_extraction.text import TfidfVectorizer
      >>> categories = ['alt.atheism', 'talk.religion.misc',
      ...               'comp.graphics', 'sci.space']
      >>> newsgroups_train = fetch_20newsgroups(subset='train',
      ...                                       categories=categories)
      >>> vectorizer = TfidfVectorizer()
      >>> vectors = vectorizer.fit_transform(newsgroups_train.data)
      >>> vectors.shape
      (2034, 34118)

    The extracted TF-IDF vectors are very sparse, with an average of 159 non-zero
    components by sample in a more than 30000-dimensional space
    (less than .5% non-zero features)::

      >>> vectors.nnz / float(vectors.shape[0])
      159.01327...

    :func:`sklearn.datasets.fetch_20newsgroups_vectorized` is a function which
    returns ready-to-use token counts features instead of file names.

    .. _`20 newsgroups website`: http://people.csail.mit.edu/jrennie/20Newsgroups/
    .. _`TF-IDF`: https://en.wikipedia.org/wiki/Tf-idf

    Filtering text for more realistic training
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

    It is easy for a classifier to overfit on particular things that appear in the
    20 Newsgroups data, such as newsgroup headers. Many classifiers achieve very
    high F-scores, but their results would not generalize to other documents that
    aren't from this window of time.

    For example, let's look at the results of a multinomial Naive Bayes classifier,
    which is fast to train and achieves a decent F-score::

      >>> from sklearn.naive_bayes import MultinomialNB
      >>> from sklearn import metrics
      >>> newsgroups_test = fetch_20newsgroups(subset='test',
      ...                                      categories=categories)
      >>> vectors_test = vectorizer.transform(newsgroups_test.data)
      >>> clf = MultinomialNB(alpha=.01)
      >>> clf.fit(vectors, newsgroups_train.target)
      MultinomialNB(alpha=0.01, class_prior=None, fit_prior=True)

      >>> pred = clf.predict(vectors_test)
      >>> metrics.f1_score(newsgroups_test.target, pred, average='macro')
      0.88213...

    (The example :ref:`sphx_glr_auto_examples_text_plot_document_classification_20newsgroups.py` shuffles
    the training and test data, instead of segmenting by time, and in that case
    multinomial Naive Bayes gets a much higher F-score of 0.88. Are you suspicious
    yet of what's going on inside this classifier?)

    Let's take a look at what the most informative features are:

      >>> import numpy as np
      >>> def show_top10(classifier, vectorizer, categories):
      ...     feature_names = np.asarray(vectorizer.get_feature_names())
      ...     for i, category in enumerate(categories):
      ...         top10 = np.argsort(classifier.coef_[i])[-10:]
      ...         print("%s: %s" % (category, " ".join(feature_names[top10])))
      ...
      >>> show_top10(clf, vectorizer, newsgroups_train.target_names)
      alt.atheism: edu it and in you that is of to the
      comp.graphics: edu in graphics it is for and of to the
      sci.space: edu it that is in and space to of the
      talk.religion.misc: not it you in is that and to of the

    You can now see many things that these features have overfit to:

    - Almost every group is distinguished by whether headers such as
      ``NNTP-Posting-Host:`` and ``Distribution:`` appear more or less often.
    - Another significant feature involves whether the sender is affiliated with
      a university, as indicated either by their headers or their signature.
    - The word "article" is a significant feature, based on how often people quote
      previous posts like this: "In article [article ID], [name] <[e-mail address]>
      wrote:"
    - Other features match the names and e-mail addresses of particular people who
      were posting at the time.

    With such an abundance of clues that distinguish newsgroups, the classifiers
    barely have to identify topics from text at all, and they all perform at the
    same high level.

    For this reason, the functions that load 20 Newsgroups data provide a
    parameter called **remove**, telling it what kinds of information to strip out
    of each file. **remove** should be a tuple containing any subset of
    ``('headers', 'footers', 'quotes')``, telling it to remove headers, signature
    blocks, and quotation blocks respectively.

      >>> newsgroups_test = fetch_20newsgroups(subset='test',
      ...                                      remove=('headers', 'footers', 'quotes'),
      ...                                      categories=categories)
      >>> vectors_test = vectorizer.transform(newsgroups_test.data)
      >>> pred = clf.predict(vectors_test)
      >>> metrics.f1_score(pred, newsgroups_test.target, average='macro')
      0.77310...

    This classifier lost over a lot of its F-score, just because we removed
    metadata that has little to do with topic classification.
    It loses even more if we also strip this metadata from the training data:

      >>> newsgroups_train = fetch_20newsgroups(subset='train',
      ...                                       remove=('headers', 'footers', 'quotes'),
      ...                                       categories=categories)
      >>> vectors = vectorizer.fit_transform(newsgroups_train.data)
      >>> clf = MultinomialNB(alpha=.01)
      >>> clf.fit(vectors, newsgroups_train.target)
      MultinomialNB(alpha=0.01, class_prior=None, fit_prior=True)

      >>> vectors_test = vectorizer.transform(newsgroups_test.data)
      >>> pred = clf.predict(vectors_test)
      >>> metrics.f1_score(newsgroups_test.target, pred, average='macro')
      0.76995...

    Some other classifiers cope better with this harder version of the task. Try
    running :ref:`sphx_glr_auto_examples_model_selection_grid_search_text_feature_extraction.py` with and without
    the ``--filter`` option to compare the results.

    .. topic:: Recommendation

      When evaluating text classifiers on the 20 Newsgroups data, you
      should strip newsgroup-related metadata. In scikit-learn, you can do this by
      setting ``remove=('headers', 'footers', 'quotes')``. The F-score will be
      lower because it is more realistic.

    .. topic:: Examples

       * :ref:`sphx_glr_auto_examples_model_selection_grid_search_text_feature_extraction.py`

       * :ref:`sphx_glr_auto_examples_text_plot_document_classification_20newsgroups.py`

```python
# Example article

newsgroups_data['data'][0]
```

    "From: lerxst@wam.umd.edu (where's my thing)\nSubject: WHAT car is this!?\nNntp-Posting-Host: rac3.wam.umd.edu\nOrganization: University of Maryland, College Park\nLines: 15\n\n I was wondering if anyone out there could enlighten me on this car I saw\nthe other day. It was a 2-door sports car, looked to be from the late 60s/\nearly 70s. It was called a Bricklin. The doors were really small. In addition,\nthe front bumper was separate from the rest of the body. This is \nall I know. If anyone can tellme a model name, engine specs, years\nof production, where this car is made, history, or whatever info you\nhave on this funky looking car, please e-mail.\n\nThanks,\n- IL\n   ---- brought to you by your neighborhood Lerxst ----\n\n\n\n\n"

```python
# Associated label

newsgroups_data['target'][0]
```

    7

```python
# Name of label

newsgroups_data['target_names'][7]
```

    'rec.autos'

```python
# Preprocessing boilerplate

n_documents = len(newsgroups_data['data'])

count_vectorizer = CountVectorizer(input='content', binary=True,
                                   max_df=0.25, min_df=1.01/n_documents) # ignore common words, words that appear once
binary_bag_of_words = count_vectorizer.fit_transform(newsgroups_data['data']) # input is a list of strings

```

```python
# Check shape

binary_bag_of_words.shape
```

    (11314, 56365)

```python
# Check that the fit has been successful

count_vectorizer.inverse_transform(binary_bag_of_words[3,:])
```

    [array(['thing', 'anyone', 'really', 'number', 'computer', 'distribution',
            'got', 'looks', 'jgreen', 'amber', 'joe', 'green', 'weitek',
            'p9000', 'harris', 'systems', 'division', '14', 'world', 'ssd',
            'csd', 'newsreader', 'tin', 'version', 'pl9', 'robert', 'kyanko',
            'rob', 'rjck', 'uucp', 'wrote', 'abraxis', 'iastate', '734340159',
            'class1', 'graphics', 'chip', 'far', 'low', 'level', 'stuff',
            'goes', 'pretty', 'nice', 'quadrilateral', 'fill', 'command',
            'requires', 'four', 'points', 'address', 'phone', 'information',
            'corporation', 'scares', 'person', 'sense', 'humor', 'jonathan',
            'winters'], dtype='<U80')]

```python
# Dict that will be useful later

inv_vocabulary = {value:key for key, value in count_vectorizer.vocabulary_.items()}
```

#### A Naive Bayes classifier for `newsgroup`

Each feature vector $x$ is a list of indicators for whether a word appears in the article. $x_i$ is 1 if the $i$th word appears, and 0 otherwise. `inv_vocabulary` matches word indices $i$ to words.

Each label $y$ is a value in $0, 1, \ldots, 19$.

The parts of a naive Bayes classifier for this problem can be summarised as:

- A probability distribution for the feature vector by class, $p(x|y = j)$ for each $j = 0, 1, \ldots, 19$. These probability distributions are assumed to have independent components: we can factorize the joint probability as a product of marginal probabilities
\begin{equation}
    p(x|y = j) = \prod_{i=1}^d p(x_i|y = j)
\end{equation}
These marginal probability distributions are Bernoulli distributions, each of which has a single parameter $\theta_{ji} := p(x_i = 1|y = j)$. This parameter is the probability of observing word $i$ in an article of class $j$.

- We will use the Laplace smoothed maximum likelihood estimate to compute these parameters. Laplace smoothing involves adding small counts to every feature for each class. Else, if a feature did not appear in the training set of a class, but then we observed it in our test data the log probability would be undefined.

- A collection of class prior probabilities $p(y = j)$. These will be set by computing the class base rates in the training set.

- A function for computing the probability of class membership via Bayes' theorem:

\begin{equation}
    p(y = j|x) = \frac{p(x|y = j)p(y = j)}{p(x)}
\end{equation}

```python
# Compute the parameter estimates (adjusted fraction of documents in class that contain word)

n_classes = newsgroups_data['target'].max() + 1
y = newsgroups_data['target']
n_words = binary_bag_of_words.shape[1]

alpha = 1e-6 # parameters for Laplace smoothing

theta = np.zeros([n_classes, n_words]) # stores parameter values - prob. word given class
for c_k in range(n_classes): # 0, 1, ..., 19
    class_mask = (y == c_k)
    N = class_mask.sum() # number of articles in class
    theta[c_k, :] = (binary_bag_of_words[class_mask, :].sum(axis=0) + alpha)/(N + alpha*2)
```

```python
theta.shape
```

    (20, 56365)

```python
# Check whether the most probable word in each class is reasonable

most_probable_word_ix = theta.argmax(axis=1) # most probable word for each class

for j, ix in enumerate(most_probable_word_ix):
    print('Most probable word in class {} is "{}".'.format(newsgroups_data['target_names'][j],
                                                           inv_vocabulary[ix]))
```

    Most probable word in class alt.atheism is "people".
    Most probable word in class comp.graphics is "graphics".
    Most probable word in class comp.os.ms-windows.misc is "windows".
    Most probable word in class comp.sys.ibm.pc.hardware is "thanks".
    Most probable word in class comp.sys.mac.hardware is "mac".
    Most probable word in class comp.windows.x is "window".
    Most probable word in class misc.forsale is "sale".
    Most probable word in class rec.autos is "car".
    Most probable word in class rec.motorcycles is "dod".
    Most probable word in class rec.sport.baseball is "he".
    Most probable word in class rec.sport.hockey is "ca".
    Most probable word in class sci.crypt is "clipper".
    Most probable word in class sci.electronics is "use".
    Most probable word in class sci.med is "reply".
    Most probable word in class sci.space is "space".
    Most probable word in class soc.religion.christian is "god".
    Most probable word in class talk.politics.guns is "people".
    Most probable word in class talk.politics.mideast is "people".
    Most probable word in class talk.politics.misc is "people".
    Most probable word in class talk.religion.misc is "he".

```python
# Define a distribution for each class

batch_of_bernoullis = tfd.Bernoulli(probs=theta)
p_x_given_y = tfd.Independent(batch_of_bernoullis,
                             reinterpreted_batch_ndims = 1)
p_x_given_y
```

    <tfp.distributions.Independent 'IndependentBernoulli' batch_shape=[20] event_shape=[56365] dtype=int32>

```python
# Take a sample of words from each class

n_samples = 10
samples = p_x_given_y.sample(n_samples)
samples.shape
```

    TensorShape([10, 20, 56365])

```python
# Choose a class

chosen_class = 15
newsgroups_data['target_names'][chosen_class]
```

    'soc.religion.christian'

```python
# Indicators for words that appear in the sample

class_sample = samples[:, chosen_class, :]
class_sample
```

    <tf.Tensor: shape=(10, 56365), dtype=int32, numpy=
    array([[0, 0, 0, ..., 0, 0, 0],
           [0, 0, 0, ..., 0, 0, 0],
           [0, 0, 0, ..., 0, 0, 0],
           ...,
           [0, 0, 0, ..., 0, 0, 0],
           [0, 0, 0, ..., 0, 0, 0],
           [0, 0, 0, ..., 0, 0, 0]])>

```python
# Perform inverse transform to test quality of fit

count_vectorizer.inverse_transform(class_sample)
```

    [array(['16', '1993', '20', '26', '29224', '34', '7415', '83', 'ac',
            'accustomed', 'additionally', 'always', 'apr', 'around', 'assume',
            'athos', 'attest', 'attributes', 'been', 'belief', 'believe',
            'belittle', 'best', 'between', 'born', 'both', 'boy', 'brother',
            'buy', 'ca', 'cases', 'charity', 'check', 'christians', 'church',
            'claim', 'clarify', 'clh', 'come', 'comes', 'consider', 'creative',
            'define', 'did', 'done', 'dumb', 'end', 'endemyr', 'enormous',
            'establish', 'everyone', 'excuse', 'existent', 'expecting',
            'experience', 'fame', 'family', 'fasting', 'faulty', 'federal',
            'follow', 'formal', 'fuzzy', 'generation', 'giving', 'god', 'good',
            'hard', 'her', 'his', 'however', 'human', 'imo', 'into', 'invited',
            'jointly', 'jonathan', 'keeping', 'knowing', 'kulikauskas', 'lies',
            'live', 'll', 'los', 'lost', 'make', 'mary', 'mdw33310', 'mean',
            'middle', 'morally', 'most', 'mother', 'motives', 'nature', 'now',
            'old', 'operator', 'opinion', 'opl', 'peace', 'people', 'perhaps',
            'pick', 'place', 'please', 'point', 'position', 'possibly',
            'pregnancy', 'presbyterian', 'president', 'probably', 'put',
            'quest', 'reconcile', 'redemption', 'regarding', 'resurrection',
            'romans', 'said', 'sas', 'school', 'science', 'sea', 'see',
            'service', 'seven', 'sexual', 'shintoism', 'since', 'sins',
            'snatch', 'society', 'sometimes', 'speaking', 'stephen',
            'subjects', 'tellabs', 'temple', 'texts', 'their', 'then', 'these',
            'third', 'time', 'top', 'traditional', 'trendy', 'truths', 'ucs',
            'ultimately', 'unanimous', 'unisa', 'vatican', 've', 'very',
            'waiting', 'want', 'were', 'why', 'woman', 'worship', 'wouldn',
            'yes', 'z5'], dtype='<U80'),
     array(['1993', '30602', '58', 'abundantly', 'acceptable', 'accident',
            'according', 'actions', 'after', 'am', 'angels', 'arlut', 'around',
            'been', 'before', 'biblical', 'birth', 'both', 'brent', 'castro',
            'catholic', 'cause', 'century', 'cgsvax', 'church', 'closing',
            'comment', 'commit', 'creation', 'credibility', 'dead', 'death',
            'deity', 'delusion', 'deserts', 'disciples', 'does', 'doesn',
            'eastern', 'either', 'emotions', 'explanation', 'famous',
            'fifteen', 'first', 'fit', 'foods', 'fundamentalism', 'general',
            'gobs', 'hartwick', 'hell', 'hi', 'hope', 'humans', 'idea',
            'incorrectly', 'issues', 'itself', 'jack', 'jasons', 'jehovah',
            'judgment', 'keeping', 'knowing', 'large', 'law', 'leaning',
            'less', 'listening', 'll', 'logic', 'mary', 'math', 'member',
            'mentorg', 'michael', 'mielke', 'misconfigured', 'misunderstand',
            'mmh', 'mother', 'name', 'newbie', 'nothingness', 'number',
            'offending', 'offering', 'ogden', 'oldest', 'others', 'own',
            'people', 'phil', 'place', 'please', 'problem', 'provide',
            'punched', 'read', 'really', 'religious', 'remark', 'reserve',
            'revelations', 'risc1', 'rob', 'rutgers', 'satanic', 'say', 'says',
            'scientific', 'sdcc13', 'seven', 'sledd', 'smack', 'someone',
            'spelled', 'stay', 'suffered', 'sweetheart', 'tbrent', 'telling',
            'terms', 'therefore', 'these', 'though', 'thus', 'touch', 'tsd',
            'udel', 'understanding', 'universe', 'unreasonable', 'us', 'use',
            'used', 'useful', 'vision', 'weak', 'why', 'yet'], dtype='<U80'),
     array(['15', '1823', '1st', '25', '34', 'accept', 'age', 'always', 'am',
            'anything', 'apart', 'apr', 'aquinas', 'arlut', 'asked',
            'attacked', 'back', 'because', 'before', 'believing', 'bible',
            'bishops', 'case', 'century', 'challenge', 'christ', 'christian',
            'civil', 'clearly', 'correct', 'could', 'course', 'cs', 'damon',
            'deadly', 'denounced', 'died', 'differently', 'discuss', 'drugs',
            'east', 'easter', 'etc', 'even', 'every', 'faith', 'father',
            'fear', 'fi', 'friends', 'gave', 'grace', 'hand', 'happen', 'he',
            'head', 'hear', 'hearing', 'heart', 'institute', 'interested',
            'involved', 'keep', 'law', 'leads', 'levitical', 'literally',
            'logic', 'luxury', 'mail', 'manner', 'men', 'molas', 'months',
            'most', 'much', 'must', 'non', 'note', 'nothing', 'noye',
            'nusunix', 'obviously', 'old', 'ordinary', 'originally', 'packed',
            'perceived', 'permanent', 'personalities', 'prayer', 'problem',
            'put', 'queen', 'quite', 'rather', 'reaction', 'reading', 'really',
            'regeneration', 'relationships', 'reply', 'rich', 'rise',
            'roboticist', 'romans', 'run', 'sas', 'save', 'scriptura', 'seems',
            'should', 'simple', 'someone', 'sometime', 'somewhere', 'son',
            'sure', 'teacher', 'then', 'these', 'three', 'through', 'times',
            'try', 'twelve', 'two', 'uga', 'understanding', 'us', 'very',
            'yet'], dtype='<U80'),
     array(['106', '40', '45', 'against', 'also', 'am', 'appear', 'apr',
            'argument', 'atheist', 'auto', 'baptized', 'because', 'being',
            'belief', 'better', 'between', 'cheek', 'christian', 'computer',
            'conceive', 'consecration', 'could', 'create', 'descendants',
            'documentary', 'does', 'each', 'ecn', 'egypt', 'endeavor', 'ends',
            'enormous', 'especially', 'etc', 'exploitation', 'fallen', 'flame',
            'follow', 'following', 'forever', 'gcx1', 'general', 'given',
            'gives', 'god', 'good', 'gotten', 'governments', 'group',
            'happening', 'he', 'health', 'hear', 'heart', 'hide', 'his', 'ic',
            'indefinitely', 'ip', 'irrevocable', 'jesus', 'jobs', 'judge',
            'july', 'kind', 'lab', 'last', 'law', 'legitimate', 'let', 'life',
            'lived', 'logic', 'lot', 'love', 'luke', 'majority', 'many', 'mi',
            'mileage', 'minds', 'missing', 'moreover', 'moses', 'muirm',
            'munns', 'need', 'nitpick', 'north', 'ones', 'origins', 'oser',
            'our', 'perfectly', 'perspective', 'petri', 'place', 'president',
            'priests', 'probably', 'professors', 'prophet', 'protestant',
            'quick', 'quotable', 'randy', 'really', 'reason', 'received',
            'recorded', 'remember', 'right', 'says', 'school', 'scripture',
            'sexually', 'should', 'someone', 'statements', 'study', 'such',
            'superceded', 'term', 'than', 'their', 'them', 'think', 'thomas',
            'those', 'thus', 'time', 'too', 'treated', 'unconditional', 'unto',
            'use', 'wanted', 'ways', 'well', 'were', 'whether', 'whom',
            'willing', 'wordprocessor', 'written', 'years', 'yes'],
           dtype='<U80'),
     array(['05', '11', '1917', '23', '26', '34', '35', 'accept', 'ancient',
            'another', 'anyway', 'appears', 'author', 'away', 'based', 'basis',
            'becuase', 'been', 'believe', 'bible', 'biblical', 'brother',
            'build', 'catholic', 'charity', 'christian', 'christians', 'col',
            'commanded', 'common', 'concept', 'contact', 'contained',
            'correct', 'currently', 'darren', 'death', 'definitely',
            'described', 'did', 'difference', 'discerning', 'doubts', 'drugs',
            'eternality', 'even', 'event', 'every', 'everything', 'excellent',
            'experiments', 'failures', 'faithful', 'fallen', 'filled',
            'follow', 'full', 'given', 'going', 'gospel', 'her', 'however',
            'indefinitely', 'insist', 'isn', 'john', 'kille', 'koberg',
            'laboratories', 'lamb', 'last', 'leave', 'life', 'll', 'may',
            'mother', 'much', 'natal', 'national', 'none', 'ogden', 'ohio',
            'optimistic', 'ourselves', 'people', 'peoples', 'personal',
            'personally', 'pius', 'post', 'pretty', 'previous', 'problem',
            'purpose', 'quantities', 'queensland', 'question', 'quotation',
            'reasons', 'receive', 'representatives', 'rigid', 'said', 'same',
            'saw', 'second', 'seem', 'seems', 'sees', 'septuagint', 'show',
            'similar', 'sisters', 'someones', 'st', 'style', 'substance',
            'thing', 'things', 'thinking', 'those', 'though', 'thought',
            'time', 'times', 'translated', 'truth', 'two', 'way', 'whereas',
            'whether', 'why', 'wilson', 'wording', 'worse', 'worship', 'write',
            'wrong', 'yes', 'yet'], dtype='<U80'),
     array(['00', '14', '18', '23', '47', 'admirable', 'after', 'again', 'ago',
            'am', 'approval', 'arguments', 'around', 'asks', 'att', 'bad',
            'ball', 'baptist', 'belief', 'believe', 'biting', 'body',
            'brought', 'candidates', 'cardinal', 'catholics', 'change',
            'christians', 'civilians', 'cleaner', 'clearly', 'collection',
            'confining', 'could', 'culture', 'daily', 'darkness', 'destroy',
            'did', 'didn', 'disobedience', 'does', 'due', 'earth', 'email',
            'employer', 'eng', 'evaluate', 'evangelical', 'even', 'everyone',
            'existence', 'expect', 'fall', 'far', 'flying', 'forward',
            'fumbling', 'gospel', 'he', 'heaven', 'hi', 'highly', 'his',
            'idea', 'identify', 'intervention', 'into', 'intrinsically',
            'january', 'jesus', 'joseph', 'learned', 'leveled', 'logic',
            'look', 'ltd', 'lumped', 'ly', 'many', 'married', 'matthew',
            'meaningless', 'meets', 'mother', 'necessarily', 'never', 'noted',
            'observation', 'older', 'originated', 'orthodox', 'otherwise',
            'oulu', 'our', 'over', 'parenthood', 'people', 'pilot',
            'politically', 'pope', 'present', 'previous', 'problem', 'regard',
            'relationship', 'remember', 'savior', 'say', 'scientific', 'sent',
            'slave', 'something', 'somewhat', 'stuff', 'sure', 'swear',
            'teaching', 'teachings', 'technologies', 'tellabs', 'testament',
            'them', 'then', 'thesis', 'thing', 'things', 'think', 'tool',
            'topic', 'transgressions', 'ucs', 'understand', 'us', 'use',
            'valid', 'very', 'viewed', 'voodoo', 'want', 'wasn', 'ways',
            'weapons', 'well', 'were'], dtype='<U80'),
     array(['00', '02', '23', '30', '31', '3422', '44', '54', '713', 'above',
            'accept', 'accomplish', 'according', 'across', 'allow', 'am',
            'animals', 'answer', 'apr', 'arguments', 'arrogance', 'assertion',
            'attack', 'attention', 'base', 'been', 'before', 'boundary',
            'brownbladerunnersugarcubeselectronicblaylockpowersspikeleekatebushhamcornpizza',
            'call', 'catholic', 'cause', 'ceremony', 'changing', 'choose',
            'christians', 'clh', 'collections', 'compared', 'converted',
            'could', 'course', 'definition', 'distinguish', 'dlecoint', 'down',
            'during', 'else', 'end', 'eos', 'eternal', 'example', 'few',
            'functioning', 'go', 'goal', 'god', 'gvg', 'harper', 'having',
            'he', 'healing', 'heard', 'here', 'hide', 'his', 'holding', 'home',
            'intended', 'into', 'jesus', 'jr', 'least', 'left', 'listen',
            'long', 'lord', 'loves', 'make', 'making', 'maridai', 'mateo',
            'mellon', 'messages', 'michael', 'midway', 'mike', 'minority',
            'minutes', 'mmalt', 'mp', 'must', 'name', 'need', 'net', 'netcom',
            'never', 'observation', 'organisation', 'overacker', 'pagan',
            'paganism', 'part', 'parts', 'people', 'person', 'peter',
            'picture', 'please', 'pleased', 'possible', 'posted', 'practice',
            'practices', 'pretty', 'proto', 'proves', 'ques', 'quoted',
            'randy', 'recite', 'reedr', 'regards', 'related', 'religious',
            'result', 'resulted', 'richard', 'right', 'rome', 'rutgers',
            'sacrifices', 'sad', 'same', 'says', 'see', 'seen', 'self',
            'situations', 'someone', 'somewhere', 'speculation', 'spirit',
            'stage', 'standard', 'stuff', 'tehy', 'telling', 'then', 'think',
            'those', 'through', 'tim', 'title', 'told', 'transfusion',
            'understanding', 'unfaithfully', 'us', 'use', 'used', 'verifying',
            'virginia', 'wanted', 'wasn', 'woman', 'yours'], dtype='<U80'),
     array(['1993', '272', '601', 'above', 'accomplish', 'alienated', 'also',
            'always', 'am', 'america', 'answers', 'anxious', 'anything',
            'around', 'arrogance', 'augustine', 'bad', 'because', 'before',
            'behave', 'believe', 'books', 'came', 'cannot', 'center', 'christ',
            'christians', 'church', 'civilization', 'come', 'correlation',
            'could', 'council', 'course', 'crucified', 'daily', 'dated',
            'deeply', 'didn', 'dreams', 'emery', 'entering', 'essential',
            'evening', 'ever', 'eye', 'falsifiable', 'found', 'gigantic',
            'god', 'good', 'having', 'he', 'here', 'hide', 'his', 'hl', 'idea',
            'ii', 'imperfections', 'importance', 'inward', 'ironic', 'italian',
            'jewish', 'judgment', 'kept', 'kille', 'knew', 'labs', 'left',
            'lent', 'level', 'limited', 'lincoln', 'look', 'lost', 'lot',
            'luck', 'made', 'make', 'marriage', 'matter', 'may', 'mean',
            'means', 'men', 'michael', 'might', 'need', 'news', 'ourselves',
            'pagan', 'participants', 'people', 'philosophies', 'power',
            'practices', 'principles', 'proof', 'proper', 'purpose', 'put',
            'quite', 'rc', 'read', 'really', 'reason', 'reply', 'rethink',
            'revelation', 'rise', 'roles', 'saying', 'second', 'sense',
            'should', 'simplified', 'speak', 'spoke', 'start', 'stephen',
            'studies', 'talents', 'talk', 'than', 'their', 'then', 'therefore',
            'think', 'third', 'thoughts', 'tod', 'treated', 'trinity',
            'trouble', 'via', 'view', 'vs', 'wise', 'woman', 'work', 'wrong',
            'years', 'yours'], dtype='<U80'),
     array(['02', '1993', '3803', 'above', 'accept', 'across', 'add',
            'alright', 'also', 'another', 'asked', 'authors', 'aziz', 'back',
            'baptist', 'becoming', 'been', 'beliefs', 'bible', 'both', 'call',
            'cease', 'cent', 'christ', 'clyde', 'commentary', 'complete',
            'corporation', 'counter', 'cure', 'declare', 'deeds', 'defense',
            'denied', 'destroying', 'die', 'died', 'discipline', 'disciplines',
            'divided', 'done', 'edged', 'end', 'enough', 'esp', 'essay',
            'established', 'faith', 'fox', 'functions', 'generally', 'given',
            'god', 'hey', 'hole', 'humble', 'imagine', 'immediately',
            'important', 'infinite', 'innocent', 'intent', 'intolerance',
            'involve', 'its', 'jasons', 'justified', 'killing', 'known',
            'larry', 'linked', 'logically', 'los', 'lot', 'make', 'making',
            'mateo', 'mean', 'members', 'midway', 'mormon', 'much', 'must',
            'name', 'new', 'nih', 'noyes', 'offshoots', 'opinion', 'over',
            'patrick', 'people', 'person', 'place', 'pretty', 'qumran',
            'quotes', 'reach', 'reaching', 'record', 'refusal', 'respects',
            'responding', 'retain', 'rope', 'rose', 'rutgers', 'said', 'scale',
            'shed', 'should', 'since', 'sins', 'skidmore', 'social',
            'somewhere', 'spacing', 'stripes', 'super', 'sure', 'tasks',
            'than', 'then', 'therefore', 'these', 'thesis', 'think', 'threads',
            'toledo', 'tsd', 'under', 'understanding', 'urbana', 've',
            'velasco', 'very', 'via', 'vituperousness', 'whatever', 'where',
            'wine', 'wing'], dtype='<U80'),
     array(['04', '08', '203', '29', '63', '81', 'africa', 'after', 'also',
            'altered', 'apocalypse', 'appropriate', 'around', 'athos',
            'attended', 'been', 'before', 'believe', 'believed', 'bible',
            'bit', 'book', 'born', 'both', 'ca', 'call', 'catholic',
            'centuries', 'chris', 'christians', 'clue', 'come', 'comes',
            'command', 'contested', 'contract', 'days', 'deciding',
            'determine', 'did', 'died', 'different', 'doctrines', 'does',
            'doubt', 'eggert', 'energy', 'engaged', 'equally', 'evils',
            'exactly', 'far', 'find', 'firmly', 'foundations', 'friday', 'god',
            'guild', 'handle', 'he', 'help', 'history', 'imho', 'included',
            'inks', 'institute', 'instructions', 'instrument',
            'interpretation', 'into', 'intolerant', 'isn', 'jews', 'jim',
            'jon', 'kiefer', 'kind', 'least', 'lie', 'list', 'listed', 'lmvec',
            'lost', 'many', 'married', 'mass', 'may', 'melinda', 'minority',
            'misc', 'mmh', 'mozart', 'mtu', 'need', 'nestorianism', 'new',
            'note', 'oppressing', 'others', 'passed', 'people', 'phrase',
            'pittsburgh', 'plan', 'possibly', 'premise', 'promised', 'quoted',
            'rather', 'reader', 'real', 'really', 'realm', 'recently',
            'recite', 'recognized', 'recognizing', 'redeemable', 'rely',
            'reply', 'represented', 'respond', 'responses', 'richard',
            'righteous', 'roman', 'salaris', 'seems', 'seen', 'sg', 'should',
            'showing', 'since', 'speak', 'specify', 'spot', 'such', 'than',
            'them', 'themselves', 'then', 'theology', 'thereby', 'these',
            'thing', 'time', 'together', 'touched', 'transgressions', 'umiacs',
            'understand', 'unfortunately', 'unnaturally', 'us', 'virtually',
            'vitalism', 'wanted', 'way', 'whosoever', 'works', 'wquinnan',
            'yet'], dtype='<U80')]

***

## Sampling and log probs

```python
# Make Multivariate Distribution

normal_distributions = tfd.MultivariateNormalDiag(loc=[[0.5, 1], [0.1, 0], [0, 0.2]],
                                 scale_diag=[[2, 3], [1, 3], [4, 4]])
normal_distributions
```

    <tfp.distributions.MultivariateNormalDiag 'MultivariateNormalDiag' batch_shape=[3] event_shape=[2] dtype=float32>

```python
# Sample

normal_distributions.sample(5)
```

    <tf.Tensor: shape=(5, 3, 2), dtype=float32, numpy=
    array([[[ 0.14818957,  0.42764258],
            [-0.28931782, -3.118083  ],
            [ 0.2619505 ,  3.1741545 ]],

           [[ 2.2465458 ,  0.52832866],
            [ 1.573347  , -2.9480422 ],
            [-4.0756607 ,  3.31636   ]],

           [[ 2.5474977 ,  1.2840819 ],
            [-0.6984171 ,  1.1638125 ],
            [ 2.4840055 ,  3.4899678 ]],

           [[ 1.6328118 ,  4.2883615 ],
            [ 2.7970288 , -2.2603824 ],
            [-6.0630474 , -9.33057   ]],

           [[ 1.4744556 , -5.195     ],
            [-0.07679487,  0.7868643 ],
            [ 2.8420002 , -1.4198972 ]]], dtype=float32)>

```python
# Multivariate Normal batched Distribution
# We are broadcasting batch shapes of `loc` and `scal_diag`
# against each other

loc = [[[0.3, 1.5, 1.], [0.2, 0.4, 2.8]],
        [[2., 2.3, 8], [1.4, 1, 1.3]]]
scale_diag = [0.4, 1., 0.7]
normal_distributions = tfd.MultivariateNormalDiag(loc=loc, scale_diag=scale_diag)
normal_distributions
```

    <tfp.distributions.MultivariateNormalDiag 'MultivariateNormalDiag' batch_shape=[2, 2] event_shape=[3] dtype=float32>

```python
# Use independent to move part of the batch shape

ind_normal_distributions = tfd.Independent(normal_distributions,
                                          reinterpreted_batch_ndims = 1)

ind_normal_distributions
```

    <tfp.distributions.Independent 'IndependentMultivariateNormalDiag' batch_shape=[2] event_shape=[2, 3] dtype=float32>

```python
# Draw some samples

samples = ind_normal_distributions.sample(5)
samples.shape # [5,2,2,3], 5: sample shape, 2: batch shape, [2,3]: event shape
```

    TensorShape([5, 2, 2, 3])

```python
# `[B, E]` shaped input

inp = tf.random.uniform([2,2,3])
ind_normal_distributions.log_prob(inp)
```

    <tf.Tensor: shape=(2,), dtype=float32, numpy=array([ -9.383299, -79.121544], dtype=float32)>

```python
# `[E]` shaped input (broadcasting over batch size)

inp = tf.random.uniform([2,3])
ind_normal_distributions.log_prob(inp)
```

    <tf.Tensor: shape=(2,), dtype=float32, numpy=array([-12.518419, -80.49691 ], dtype=float32)>

```python
#`[S, B, E]` shaped input (broadcasting over samples)

inp = tf.random.uniform([9,2, 2,3])
ind_normal_distributions.log_prob(inp)
```

    <tf.Tensor: shape=(9, 2), dtype=float32, numpy=
    array([[ -7.8732643, -83.68565  ],
           [ -8.782089 , -68.17819  ],
           [-11.045224 , -68.93258  ],
           [-13.482121 , -75.4684   ],
           [-10.517141 , -79.20174  ],
           [-11.771085 , -79.274345 ],
           [-11.529003 , -76.29223  ],
           [-10.045839 , -69.15516  ],
           [ -9.47405  , -83.97304  ]], dtype=float32)>

```python
# `[S, b, e]` shaped input, where [b, e] is broadcastable over [B, E]

inp = tf.random.uniform([5, 1, 2, 1]) # batch shape = 1, broadcasted to 2, event shape = [2,1], broadcasted to [2,3]
ind_normal_distributions.log_prob(inp)
```

    WARNING: Nested component "MultivariateNormalDiag_scale_matvec_linear_operator" in composition "MultivariateNormalDiag_chain_of_MultivariateNormalDiag_shift_of_MultivariateNormalDiag_scale_matvec_linear_operator" operates on inputs with increased degrees of freedom. This may result in an incorrect log_det_jacobian.

    <tf.Tensor: shape=(5, 2), dtype=float32, numpy=
    array([[ -9.102527, -68.08513 ],
           [-10.051434, -81.062515],
           [ -9.239349, -64.17423 ],
           [-11.851111, -87.30027 ],
           [ -9.25128 , -73.067276]], dtype=float32)>

#### Naive Bayes example

Lets now use what we have learned and continue the Naive Bayes classifier we were building last tutorial.

```python
from sklearn.datasets import fetch_20newsgroups
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.naive_bayes import BernoulliNB
from sklearn.metrics import f1_score
```

```python
# Making a function get_data which:
#   1) Fetches the 20 newsgroup dataset
#   2) Performs a word count on the articles and binarizes the result
#   3) Returns the data as a numpy matrix with the labels

def get_data(categories):

    newsgroups_train_data = fetch_20newsgroups(data_home='20_Newsgroup_Data/',
                                               subset='train', categories=categories)
    newsgroups_test_data = fetch_20newsgroups(data_home='20_Newsgroup_Data/',
                                              subset='test', categories=categories)

    n_documents = len(newsgroups_train_data['data'])
    count_vectorizer = CountVectorizer(input='content', binary=True,max_df=0.25, min_df=1.01/n_documents)

    train_binary_bag_of_words = count_vectorizer.fit_transform(newsgroups_train_data['data'])
    test_binary_bag_of_words = count_vectorizer.transform(newsgroups_test_data['data'])

    return (train_binary_bag_of_words.todense(), newsgroups_train_data['target']),  (test_binary_bag_of_words.todense(), newsgroups_test_data['target'])
```

```python
# Defining a function to conduct Laplace smoothing. This adds a base level of probability for a given feature
# to occur in every class.

def laplace_smoothing(labels, binary_data, n_classes):
    # Compute the parameter estimates (adjusted fraction of documents in class that contain word)
    n_words = binary_data.shape[1]
    alpha = 1 # parameters for Laplace smoothing
    theta = np.zeros([n_classes, n_words]) # stores parameter values - prob. word given class
    for c_k in range(n_classes): # 0, 1, ..., 19
        class_mask = (labels == c_k)
        N = class_mask.sum() # number of articles in class
        theta[c_k, :] = (binary_data[class_mask, :].sum(axis=0) + alpha)/(N + alpha*2)

    return theta
```

```python
# Getting a subset of the 20 newsgroup dataset

categories = ['alt.atheism', 'talk.religion.misc', 'comp.graphics', 'sci.space']

(train_data, train_labels), (test_data, test_labels) = get_data(categories=categories)
smoothed_counts = laplace_smoothing(labels=train_labels, binary_data=train_data, n_classes=len(categories))
```

```python
train_data.shape, train_labels.shape
```

    ((2034, 17495), (2034,))

To now make our NB classifier we need to build three functions:
* Compute the class priors
* Build our class conditional distributions
* Put it all together and classify our data

```python
# Function which computes the prior probability of every class based on frequency of occurence in
# the dataset

def class_priors(n_classes, labels):
    counts = np.zeros(n_classes)
    for c_k in range(n_classes):
        counts[c_k] = np.sum(np.where(labels==c_k, 1, 0))
    priors = counts / np.sum(counts)
    print('The class priors are {}'.format(priors))
    return priors
```

```python
# Run the function

priors = class_priors(n_classes = len(categories), labels = train_labels)
```

    The class priors are [0.2359882  0.28711898 0.29154376 0.18534907]

```python
# Now we will do a function that given the feature occurence counts returns a Bernoulli distribution of
# batch_shape=number of classes and event_shape=number of features.

def make_distribution(probs):
    batch_of_bernoullis = tfd.Bernoulli(probs=probs)
    dist = tfd.Independent(batch_of_bernoullis,
                          reinterpreted_batch_ndims = 1)

    return dist

tf_dist = make_distribution(smoothed_counts)
tf_dist
```

    <tfp.distributions.Independent 'IndependentBernoulli' batch_shape=[4] event_shape=[17495] dtype=int32>

```python
# The final function predict_sample which given the distribution, a test sample, and the class priors:
#   1) Computes the class conditional probabilities given the sample
#   2) Forms the joint likelihood
#   3) Normalises the joint likelihood and returns the log prob

def predict_sample(dist, sample, priors):
    cond_probs=dist.log_prob(sample)
    joint_likelihood = tf.add(np.log(priors), cond_probs)
    norm_factor = tf.math.reduce_logsumexp(joint_likelihood, axis = -1, keepdims = True)
    log_prob = joint_likelihood - norm_factor

    return log_prob
```

#### Computing log_probs

```python
# Predicting one example from our test data

log_probs = predict_sample(tf_dist, test_data[0], priors)
log_probs
```

    <tf.Tensor: shape=(4,), dtype=float32, numpy=
    array([-6.1706207e+01, -1.5258789e-05, -1.1600113e+01, -6.3319962e+01],
          dtype=float32)>

```python
# Loop over our test data and classify.

probabilities = []
for sample, label in zip(test_data, test_labels):
    probabilities.append(tf.exp(predict_sample(tf_dist, sample, priors)))

probabilities = np.asarray(probabilities)
predicted_classes = np.argmax(probabilities, axis =-1)
print('f1 ', f1_score(test_labels, predicted_classes, average='macro'))
```

    f1  0.7837248814543706

```python
# Make a Bernoulli Naive Bayes classifier using sklearn with the same level of alpha smoothing.

clf = BernoulliNB(alpha=1)
clf.fit(train_data, train_labels)
pred = clf.predict(test_data)
print('f1 from sklean ', f1_score(test_labels, pred, average='macro'))
```

    f1 from sklean  0.7848499112849504

***

## Trainable Distributions

```python
from sklearn.datasets import fetch_20newsgroups
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.naive_bayes import BernoulliNB
from sklearn.metrics import f1_score
```

```python
# Define an exponential distribution

exponential = tfd.Exponential(rate=0.3, name = 'exp')
```

```python
# Plot

plt.hist(exponential.sample(5000), bins=100, density=True)
plt.show()
```

```python
# Define an exponential distribution with a trainable rate parameter

exp_train = tfd.Exponential(rate = tf.Variable(1., name = 'rate'),
                           name = 'exp_train')

exp_train.trainable_variables
```

    (<tf.Variable 'rate:0' shape=() dtype=float32, numpy=1.0>,)

```python
# Define the negative log likelihood

def nll(x_train, distribution):
    return -tf.reduce_mean(distribution.log_prob(x_train))
```

```python
# Define a function to compute the loss and gradients

@tf.function
def get_loss_and_grads(x_train, distribution):

    with tf.GradientTape() as tape:
        tape.watch(distribution.trainable_variables) # Watch whichever whose gradient is taken with respect to
        loss = nll(x_train, distribution) # Get the loss
        grads = tape.gradient(loss, distribution.trainable_variables) # Calculate the gradient

    return loss, grads
```

```python
# Optimize

def exponential_dist_optimisation(data, distribution):

    # Keep results for plotting
    train_loss_results = []
    train_rate_results = []

    optimizer = tf.keras.optimizers.SGD(learning_rate=0.05)

    num_steps = 10

    for i in range(num_steps):
        loss, grads = get_loss_and_grads(data, distribution)
        optimizer.apply_gradients(zip(grads, distribution.trainable_variables))

        rate_value = distribution.rate.value()
        train_loss_results.append(loss)
        train_rate_results.append(rate_value)
        print("Step {:03d}: Loss: {:.3f}: Rate: {:.3f}".format(i, loss, rate_value))

    return train_loss_results, train_rate_results
```

```python
# Get some data and train

sampled_data = exponential.sample(5000)

train_loss_results, train_rate_results = exponential_dist_optimisation(data = sampled_data,
                                                                      distribution = exp_train)
```

    Step 000: Loss: 3.111: Rate: 0.769
    Step 001: Loss: 2.867: Rate: 0.665
    Step 002: Loss: 2.659: Rate: 0.571
    Step 003: Loss: 2.493: Rate: 0.489
    Step 004: Loss: 2.371: Rate: 0.422
    Step 005: Loss: 2.292: Rate: 0.371
    Step 006: Loss: 2.248: Rate: 0.337
    Step 007: Loss: 2.228: Rate: 0.316
    Step 008: Loss: 2.222: Rate: 0.305
    Step 009: Loss: 2.220: Rate: 0.300

```python
# Predicted value for the rate parameter

pred_value = exp_train.rate.numpy()
exact_value = exponential.rate.numpy()

print("Exact rate: ", exact_value)
print("Pred rate:  ", pred_value)
```

    Exact rate:  0.3
    Pred rate:   0.2996781

```python
# Plot to see the convergence of the estimated and true parameters

tensor_exact_value = tf.constant(exact_value, shape=[len(train_rate_results)])

fig, axes = plt.subplots(2, sharex=True, figsize=(12, 8))
fig.suptitle('Convergence')

axes[0].set_ylabel("Loss", fontsize=14)
axes[0].plot(train_loss_results)

axes[1].set_ylabel("Rate", fontsize=14)
axes[1].set_xlabel("Epoch", fontsize=14)
axes[1].plot(train_rate_results, label='trainable rate variable')
axes[1].plot(tensor_exact_value, label='exact rate')
axes[1].legend()
plt.show()
```

```python
# Making a function get_data which:
#   1) Fetches the 20 newsgroup dataset
#   2) Performs a word count on the articles and binarizes the result
#   3) Returns the data as a numpy matrix with the labels

def get_data(categories):

    newsgroups_train_data = fetch_20newsgroups(data_home='20_Newsgroup_Data/',
                                               subset='train', categories=categories)
    newsgroups_test_data = fetch_20newsgroups(data_home='20_Newsgroup_Data/',
                                              subset='test', categories=categories)

    n_documents = len(newsgroups_train_data['data'])
    count_vectorizer = CountVectorizer(input='content', binary=True,max_df=0.25, min_df=1.01/n_documents)
    train_binary_bag_of_words = count_vectorizer.fit_transform(newsgroups_train_data['data'])
    test_binary_bag_of_words = count_vectorizer.transform(newsgroups_test_data['data'])

    return (train_binary_bag_of_words.todense(), newsgroups_train_data['target']),  (test_binary_bag_of_words.todense(), newsgroups_test_data['target'])

```

```python
# Defining a function to conduct laplace smoothing. This adds a base level of probability for a given feature
# to occur in every class.

def laplace_smoothing(labels, binary_data, n_classes):
    # Compute the parameter estimates (adjusted fraction of documents in class that contain word)
    n_words = binary_data.shape[1]
    alpha = 1 # parameters for Laplace smoothing
    theta = np.zeros([n_classes, n_words]) # stores parameter values - prob. word given class
    for c_k in range(n_classes): # 0, 1, ..., 19
        class_mask = (labels == c_k)
        N = class_mask.sum() # number of articles in class
        theta[c_k, :] = (binary_data[class_mask, :].sum(axis=0) + alpha)/(N + alpha*2)

    return theta
```

```python
# Now we will do a function that given the feature occurence counts returns a Bernoulli distribution of
# batch_shape=number of classes and event_shape=number of features.

def make_distributions(probs):
    batch_of_bernoullis = tfd.Bernoulli(probs=probs) # shape (n_classes, n_words)
    dist = tfd.Independent(batch_of_bernoullis, reinterpreted_batch_ndims=1)
    return dist
```

```python
# Function which computes the prior probability of every class based on frequency of occurence in
# the dataset

def class_priors(n_classes, labels):
    counts = np.zeros(n_classes)
    for c_k in range(n_classes):
        counts[c_k] = np.sum(np.where(labels==c_k, 1, 0))
    priors = counts / np.sum(counts)
    print('The class priors are {}'.format(priors))
    return priors
```

```python
# The final function predict_sample which given the distribution, a test sample, and the class priors:
#   1) Computes the class conditional probabilities given the sample
#   2) Forms the joint likelihood
#   3) Normalises the joint likelihood and returns the log prob

def predict_sample(dist, sample, priors):
    cond_probs = dist.log_prob(sample)
    joint_likelihood = tf.add(np.log(priors), cond_probs)
    norm_factor = tf.math.reduce_logsumexp(joint_likelihood, axis=-1, keepdims=True)
    log_prob = joint_likelihood - norm_factor

    return log_prob
```

```python
# Now we learn the distribution using gradient tape

def make_distribution_withGT(data, labels, nb_classes):

    class_data = []
    train_vars = []
    distributions = []
    for c in range(nb_classes):
        train_vars.append(tf.Variable(initial_value=np.random.uniform(low=0.01, high =0.1, size=data.shape[-1])))
        distributions.append(tfd.Bernoulli(probs=train_vars[c]))
        class_mask = (labels == c)
        class_data.append(data[class_mask, :])

    for c_num in range(0,nb_classes):
        optimizer = tf.keras.optimizers.Adam()
        print('\n%-------------------%')
        print('Class ', c_num)
        print('%-------------------%')

        for i in range(0,100):
            loss, grads = get_loss_and_grads(class_data[c_num], distributions[c_num])

            if i % 10 == 0:
                print(f"iter: {i}, loss: {loss}")

            optimizer.apply_gradients(zip(grads, distributions[c_num].trainable_variables))

            eta = 1e-3
            clipped_probs = tf.clip_by_value(distributions[c_num].trainable_variables,
                                               clip_value_min = eta, clip_value_max = 1)
            train_vars[c_num] = tf.squeeze(clipped_probs)

    dist = tfd.Bernoulli(probs=train_vars)
    dist = tfd.Independent(dist,reinterpreted_batch_ndims=1)

    print(dist)

    return dist

```

```python
# Make the same Naive Bayes classifier we did last tutorial

categories = ['alt.atheism', 'talk.religion.misc', 'comp.graphics', 'sci.space']

(train_data, train_labels), (test_data, test_labels) = get_data(categories)

smoothed_counts = laplace_smoothing(labels=train_labels, binary_data=train_data, n_classes=len(categories))

priors = class_priors(n_classes=len(categories), labels=train_labels)
tf_dist = make_distributions(smoothed_counts)
```

    The class priors are [0.2359882  0.28711898 0.29154376 0.18534907]

```python
# Now train the distributions with gradient tape

GT_dist = make_distribution_withGT(data=train_data, labels=train_labels, nb_classes=4)
```

    %-------------------%
    Class  0
    %-------------------%
    iter: 0, loss: 0.07818863338448928
    iter: 10, loss: 0.06884439343409732
    iter: 20, loss: 0.06013162110633655
    iter: 30, loss: 0.052063324788220046
    iter: 40, loss: 0.04460339684834779
    iter: 50, loss: 0.03771084382535268
    iter: 60, loss: 0.03134889347581688
    iter: 70, loss: 0.02546313356560098
    iter: 80, loss: 0.020004306615703942
    iter: 90, loss: 0.014918871875489001

    %-------------------%
    Class  1
    %-------------------%
    iter: 0, loss: 0.07171624343963623
    iter: 10, loss: 0.06236624065740422
    iter: 20, loss: 0.05357182165652491
    iter: 30, loss: 0.04540698374636691
    iter: 40, loss: 0.03782469970343827
    iter: 50, loss: 0.03080130935697501
    iter: 60, loss: 0.024314267545893557
    iter: 70, loss: 0.01833428770661267
    iter: 80, loss: 0.01280623158469168
    iter: 90, loss: 0.007697548795524027

    %-------------------%
    Class  2
    %-------------------%
    iter: 0, loss: 0.07858428526794405
    iter: 10, loss: 0.06950023923581672
    iter: 20, loss: 0.06113629997359445
    iter: 30, loss: 0.05350522080589954
    iter: 40, loss: 0.046559974362836716
    iter: 50, loss: 0.04026579419099047
    iter: 60, loss: 0.03457269969985814
    iter: 70, loss: 0.029442899700768924
    iter: 80, loss: 0.024841023768524005
    iter: 90, loss: 0.020697316056650004

    %-------------------%
    Class  3
    %-------------------%
    iter: 0, loss: 0.07967133754049358
    iter: 10, loss: 0.07040230489589971
    iter: 20, loss: 0.06177910918633154
    iter: 30, loss: 0.053786659224856585
    iter: 40, loss: 0.04637528067393571
    iter: 50, loss: 0.039513359681616934
    iter: 60, loss: 0.03315929894524425
    iter: 70, loss: 0.027258416997556188
    iter: 80, loss: 0.02178151961279436
    iter: 90, loss: 0.016665526899949986
    tfp.distributions.Independent("IndependentBernoulli", batch_shape=[4], event_shape=[17495], dtype=int32)

```python
# Compare the two results

for dist in [GT_dist,tf_dist]:
    probabilities = []
    for sample, label in zip(test_data, test_labels):
        probabilities.append(predict_sample(dist, sample, priors))

    probabilities = np.asarray(probabilities)
    predicted_classes = np.argmax(probabilities, axis =-1)
    print('f1 ', f1_score(test_labels, predicted_classes, average='macro'))
```

    f1  0.09539473684210525
    f1  0.7837248814543706

```python

```

Probabilistic Deep Learning with TensorFlow 2/Coding Tutorial W1.ipynb


# Ensemble Learning

* The basic idea of ensemble learning is to have multiple learning algorithms for the same problem and combine their results to make a final prediction

* There are multiple types on ensemble learning.  Common approaches include:
    * Boosting
    * Bagging/Bootstrapping
    * Random Forests
    * Mixture of Experts

## Boosting and Bagging

* When you have one data set, usually you may train an algorithm and learn a single set of parameters.  However, when we do this, we have no idea how stable/variable those parameters that we estimated are.
* Bootstrapping can show us the variation in estimated parameter values given a particular data set. Sometimes, it can also help to improve our predictions.
* Essentially, to perform bootstrapping, you sample from your data set *with replacement* and train your algorithm to estimate the parameters with each sampled subset.  You can then look at how much the parameters vary with each sampled subset and you can also combine your estimates from each trained method by averaging over all of the results for regression:
\begin{equation}
y_{com}(\mathbf{x}) = \frac{1}{M} \sum_{m=1}^M y_m(\mathbf{x})
\end{equation}
* You can aggregate results over all your bootstrap samples using majority vote for classification.

```python
import numpy as np
import matplotlib.pyplot as plt
import math
import textwrap
%matplotlib inline

def generateRandData(N, l, u, gVar):
	'''generateRandData(N, l, u, gVar): Generate N uniformly random data points in the range [l,u) with zero-mean Gaussian random noise with variance gVar'''
	x = np.random.uniform(l,u,N)
	e = np.random.normal(0,gVar,N)
	t = np.sin(2*math.pi*x) + e
	return x,t

def fitdataReg(x,t,M,la):
	'''fitdata(x,t,M): Fit a polynomial of order M to the data (x,t)'''
	X = np.array([x**m for m in range(M+1)]).T
	w = np.linalg.inv(X.T@X+(la*np.identity(M+1)))@X.T@t
	return w

def plotPoly(x,t,xrange, y, esty, subplotloc,la=0):
	#plot everything
	plt.subplot(*subplotloc) #identify the subplot to use
	# plt.tight_layout()
	plt.ylim([-2,2])
	p1 = plt.plot(xrange, y, 'g') #plot true value
	p2 = plt.plot(x, t, 'bo') #plot training data
	p3 = plt.plot(xrange, esty, 'r') #plot estimated value

	#add title, legend and axes labels
	plt.ylabel('t') #label x and y axes
	plt.xlabel('x')

def bootstrapRegression(M, numData,percentSample,numSamples):

    #generate data
	x,t = generateRandData(numData,0,1,1)
	numDataSamples = round(percentSample*numData)
	subplotloc = [2, round(numSamples/2), 1]
	fig = plt.figure()
	xrange = np.arange(0.05,.95,0.001)  #get equally spaced points in the xrange
	esty = np.empty([numSamples, xrange.shape[0]])

	for iter in range(numSamples):
        #select a random subset of the data
		rp = np.random.permutation(numData)
		x_sub = x[rp[0:numDataSamples-1]]
		t_sub = t[rp[0:numDataSamples-1]]

        #fit the random subset
		w = fitdataReg(x_sub,t_sub,M,0)

        #plot results
		subplotloc[2] = iter+1
		y = np.sin(2*math.pi*xrange) #compute the true function value
		X = np.array([xrange**m for m in range(w.shape[0])]).T
		esty[iter,:] = X@w #compute the predicted value
		plotPoly(x_sub,t_sub,xrange,y,esty[iter,:],subplotloc)

    #combine the bootstrapped results
	comy = esty.mean(0)
	yerr = esty.var(0)

    # compare to full data set
	fig = plt.figure()
	plotPoly(x,t,xrange,y,comy,[1, 1, 1])
	plt.errorbar(xrange, comy, yerr=yerr, fmt='r.',ms=10,errorevery=10)

	fig = plt.figure()
	w = fitdataReg(x,t,M,0)
	y = np.sin(2*math.pi*xrange) #compute the true function value
	X = np.array([xrange**m for m in range(w.shape[0])]).T
	yy = X@w #compute the predicted value
	plotPoly(x,t,xrange,y,yy, [1, 1, 1])

#Figure 1.7 from text
bootstrapRegression(5, 50,.75,20)

```

# Boosting: AdaBoost

* Goal:  Combine base (``weak'') classifiers to form a committee whose performance is better than any of the single base classifiers.
* The base classifiers are trained in sequence (not in parallel like in bootstrapping)
* Each base classifier is trained using a weighted data set (different weights for each base classifier)
* Points that are misclassified by a base classifier are weighted more heavily while training the next base classifier

* Consider a two class classification problem with $\mathbf{X} = \left\{ \mathbf{x}_1, \mathbf{x}_2, \ldots, \mathbf{x}_n\right\}$ with corresponding labels $y_i \in \left\{ -1,1\right\}$.
* The goal is to construct a classifier of the form:
\begin{equation}
f(\mathbf{x}) = sign(F(\mathbf{x}))
\end{equation}
where
\begin{equation}
F(\mathbf{x}) = \sum_{k=1}^K \frac{1}{2}\alpha_k \phi(\mathbf{x}; \theta_k)
\end{equation}
where $\phi(\mathbf{x}; \theta_k)$ is the base classifier.

* We need to determine the parameter values for each base classifier:
\begin{eqnarray}
\arg \min_{\alpha_k, \theta_k} \sum_{i=1}^N \exp\left(-y_i F(\mathbf{x}_i) \right)
\end{eqnarray}
* This cost function penalizes the samples that are incorrectly classified ($y_iF(\mathbf{x}_i) < 0$) heavily
* Direct optimization of all $\alpha$s and $\theta$s is difficult.   So, we iteratively optimize (which is sub-optimal). At each stage, we train one base classifier holding fixed all those that have already been trained.

* Let:
\begin{eqnarray}
F_m(\mathbf{x}) &=& \sum_{k=1}^m \frac{1}{2}\alpha_k \phi(\mathbf{x}; \theta_k)\\
&=& F_{m-1}(\mathbf{x}) + \frac{1}{2}\alpha_m \phi(\mathbf{x}; \theta_m)
\end{eqnarray}
* At step $m$, we optimize for $\alpha_m$ and $\theta_m$ where $F_{m-1}(\mathbf{x})$ is fixed:
\begin{eqnarray}
(\alpha_m, \theta_m) &=& \arg \min_{\alpha, \theta} J(\alpha, \theta)\\
&=& \arg \min_{\alpha, \theta} \sum_{i=1}^N \exp\left( -y_i\left( F_{m-1}(\mathbf{x}_i) +\frac{1}{2} \alpha\phi(\mathbf{x}_i; \theta)\right)\right)
\end{eqnarray}
* So, let's optimize this in two steps: first $\theta_m$ and then $\alpha_m$
\begin{eqnarray}
\theta_m &=&  \arg \min_{\theta} \sum_{i=1}^N \exp\left( -y_i\left( F_{m-1}(\mathbf{x}_i) + \frac{1}{2}\alpha\phi(\mathbf{x}_i; \theta)\right)\right)\\
&=& \arg \min_{\theta} \sum_{i=1}^N w_i^{(m)} \exp\left( -\frac{1}{2}y_i\alpha\phi(\mathbf{x}_i; \theta)\right)
\end{eqnarray}
where
\begin{equation}
w_i^{(m)} = \exp\left(-y_iF_{m-1}(\mathbf{x}_i)\right)
\end{equation}

* This can be re-written as:

\begin{eqnarray}
\theta_m &=&  \arg \min_{\theta} \exp\left(-\alpha_m/2\right)\sum_{n \in T_m}w_n^{(m)} + \exp\left(\alpha_m/2\right)\sum_{n \in M_m}w_n^{(m)} \nonumber \\
&=& \left( \exp\left(\alpha_m/2\right) - \exp\left(-\alpha_m/2\right)\right)\sum_{i=1}^Nw_i^{(m)} I(\phi_m(\mathbf{x}_i;\theta) \ne y_i) + \exp\left(-\alpha_m/2\right)\sum_{i=1}^Nw_i^{(m)}
\end{eqnarray}

* This is equivalent to minimizing

\begin{equation}
\arg \min_{\theta} \sum_{i=1}^N w_i^{(m)} I(\phi_m(\mathbf{x}_i;\theta) \ne y_i)
\end{equation}

* Once we have the optimal classifier at step $m$ (i.e., $\theta_m$), then we determine the $\alpha_m$ values
\begin{eqnarray}
\sum_{y_i\phi(\mathbf{x}_i;\theta_m)<0}w_i^{(m)} = P_m\\
\sum_{y_i\phi(\mathbf{x}_i;\theta_m)>0}w_i^{(m)} = 1 - P_m
\end{eqnarray}
* Plugging this into J, we get:
\begin{eqnarray}
\alpha_m = \arg\min_{\alpha} \left\{ \exp(-\alpha)(1-P_m) + \exp(\alpha)P_m\right\}
\end{eqnarray}
* Take the derivative with respect to $\alpha$, set to zero, we get:
\begin{equation}
\alpha_m = \frac{1}{2}\ln\frac{1-P_m}{P_m}
\end{equation}
* Once you get $\theta_m$ and $\alpha_m$, you compute the weights for the next step:
\begin{equation}
w_i^{(m+1)} = \frac{\exp(-y_iF_m(\mathbf{x}_i))}{Z_m} = \frac{\exp(-y_i\alpha_m\phi(\mathbf{x}_i;\theta_m))}{Z_m}
\end{equation}
where
\begin{equation}
Z_m = \sum_{i=1}^N w_i^{(m)}\exp\left(-y_i\alpha_m\phi(\mathbf{x}_i;\phi_m)\right)
\end{equation}
* Notice that the weight corresponding to a sample is increased (or decreased) with respect to its value in the previous iteration
* Notice that the amount of increase or decrease depends on $\alpha_m$ which controls the relative importance of the $m^{th}$ term in building up the final classifier

```python

```

## Random Forests
* A forest is made up of many trees...
* For classification/regression, put an input vector down each of the trees in the forest.  For classification, classify the data point using majority vote.  For regression, average the values
* Each tree is grown using:
    * Sample $N$ data points (with replacement, i.e., a bootstrap sample) from the full training data set
    * Specify a number $d << D$.  $d$ variables are selected at random out of all $D$ features to determine the split on the node.  Select the best of the $d$ features to split at that node
    * Grow each tree as much as possible (i.e., no pruning or stopping early)

* Error relates to correlation between the trees.  Greater correlation leads to greater error.  *Does this make sense?*
* Error also relates to the strength of each individual tree.  Better individual trees lead to lower error
* https://www.stat.berkeley.edu/~breiman/RandomForests/cc_home.htm

# Dropout

* This is a method to help prevent overfitting and regularize a network.
* The approach attempts to minimize co-dependencies between neurons and enhance robustness of network
* Dropout has one parameter $p$.  In each iteration, you randomly exclude each neuron with probability $1-p$ during the training pass (in both forward and backward propagation).  Each iteration, you resample which neurons to keep and which to dropout.
* Dropout is related to the concept of ensemble learning with the unique case that the various models in the ensemble share parameters and these models are "combined" into a single model/network at test as opposed to training a fusion model or doing a simple average between outputs.
* During test, you use all neurons all the time.
* Please see and read: https://www.cs.toronto.edu/~hinton/absps/JMLRdropout.pdf

```python

```


#  Industry Entry-Exit Model

**Randall Romero Aguilar, PhD**

This demo is based on the original Matlab demo accompanying the  Computational Economics and Finance 2001 textbook by Mario Miranda and Paul Fackler.

Original (Matlab) CompEcon file: **demdp03.m**

Running this file requires the Python version of CompEcon. This can be installed with pip by running

    !pip install compecon --upgrade

Last updated: 2021-Oct-01

## About

A firm operates in an uncertain profit environment. At the beginning of each period, the firm observes its potential short-run operating profit over the coming period $\pi$, which may be negative, and decides whether to operate, making a short-run profit $\pi$, or not operate, making a short-run profit $0$. Although the firm faces no fixed costs, it incurs a shutdown cost $K_0$ when it closes and a start-up cost $K_1$ when it reopens. The short-run profit $\pi$ is an exogenous continuous-valued Markov process
\begin{equation}
    \pi_{t+1} = h(\pi_t, \epsilon_{t+1})
\end{equation}

**What is the optimal entry-exit policy?** In particular, how low must the short-run profit be for an operating firm to close, and how high must the short-run profit be for nonoperating firm to reopen?

This is an infinite horizon, stochastic model with time $t$ measured in years. The state variables
\begin{align}
    \pi &\in (−\infty,\infty)\\
    d &\in \{0, 1\}
\end{align}

are the current short-run profit, a continuous variable, and the operational status of the firm, a binary variable that equals 1 if the firm is operating and 0 if the firm is not operating. The choice variable
\begin{equation}
    j \in \{0, 1\}
\end{equation}

is the operating decision for the coming year, a binary variable that equals 1 if the firm operates and 0 if does not operate. The state transition function is
\begin{equation}
    g(\pi, d, j, \epsilon) = h(\pi, \epsilon), j)
\end{equation}

The reward function is
\begin{equation}
    f(\pi, d, j) = \pi j − K_1(1 − d)j − K_0 d(1 − j)
\end{equation}

The value of the firm, given that the current short-run profit is $\pi$ and the firm's operational status is $d$, satisfies the Bellman equation
\begin{equation}
    V(\pi,d) = \max_{j=0,1}\left\{\pi j − K_1(1−d)j − K_0d(1−j) + \delta E_\epsilon V\left(h(\pi,\epsilon),j\right)\right\}
\end{equation}

```python
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from compecon import BasisSpline, DPmodel,qnwnorm, demo
```

### The reward function

The reward function is
\begin{equation}
    f(\pi, d, j) = \pi j − K_1(1 − d)j − K_0 d(1 − j)
\end{equation}

 where the exit cost is $K_0=0$ and the entry cost is $K_1=10$.

```python
K0     = 0.0
K1     = 10

def profit(p, x, d, j):
    return p * j - K1 * (1 - d) * j - K0 * d * (1 - j)
```

### The transition function

Assuming that the short-run profit $\pi$ is an exogenous Markov process
\begin{equation}
    \pi_{t+1} = g(\pi_t,\epsilon_{t+1}) = \bar{\pi} + \gamma(\pi_t − \bar{\pi}) + \epsilon_{t+1}
\end{equation}
where $\bar{\pi}=1.0$ and $\gamma=0.7$.

```python
pbar   = 1.0
gamma  = 0.7

def transition(p, x, d, j, in_, e):
    return pbar + gamma * (p - pbar) + e
```

In the transition function $\epsilon_t$ is an i.i.d. normal(0, $σ^2$), with $\sigma=1$. We discretize this distribution by using a discrete distribution, matching the first 10 moments of the normal distribution.

```python
m = 5  # number of profit shocks
sigma  = 1.0
[e,w] = qnwnorm(m,0,sigma **2)
```

The collocation method calls for the analyst to select $n$ basis functions $\varphi_j$ and $n$ collocation nodes $(\pi_i,d_i)$, and form the value function approximant $V(\pi,d) \approx \sum_{j=1}^{n} c_j\varphi_j(\pi,d)$ whose coefficients $c_j$ solve the collocation equation

\begin{equation}
    \sum_{j=1}^{n} c_j\varphi_j(\pi_i,d_i) = \max_{x\in\{0,1\}}\left\{\pi_i x − K_1(1−d_i)x − K_0 d_i(1−x) + \delta\sum_{k=1}^{m}\sum_{j=1}^{n}w_k c_j \varphi_j(\hat{\pi}_{ik},x)\right\}
\end{equation}

where $\hat\pi_{ik}=g(\pi_i,\epsilon_k)$ and where $\epsilon_k$ and $w_k$ represent quadrature nodes and weights for
the normal shock.

For the approximation, we use a cubic spline basis with $n=250$ nodes between $p_{\min}=-20$ and $p_\max=20$.

```python
n = 250
pmin = -20
pmax =  20
basis = BasisSpline(n, pmin, pmax, labels=['profit'])
print(basis)
```

    A 1-dimension Cubic spline basis:  using 250 Canonical nodes and 250 polynomials
    ___________________________________________________________________________
    	profit: 250 nodes in [-20.00,  20.00]

    ===========================================================================
    WARNING! Class Basis is still work in progress

Discrete states and discrete actions are

```python
dstates = ['idle', 'active']
dactions = ['close', 'open']
```

The Bellman equation is represeted by a ```DPmodel``` object, where we assume a discount factor of $\delta=0.9$. Notice that the discrete state transition is deterministic, with transition matrix
\begin{equation}
    h=\begin{bmatrix}0&0\\1&1\end{bmatrix}
\end{equation}

```python
model = DPmodel(basis,
                profit, transition,
                i=dstates,
                j=dactions,
                discount=0.9, e=e, w=w,
                h=[[0, 0], [1, 1]])
```

## SOLUTION

To solve the model, we simply call the ```solve``` method on the ```model``` object.

```python
S = model.solve(show=True)
S.head()
```

    Solving infinite-horizon model collocation equation by Newton's method
    iter change       time
    ------------------------------
       0       6.0e+01    0.0519
       1       3.2e+01    0.1087
       2       2.9e+00    0.1606
       3       4.3e-01    0.2154
       4       4.9e-14    0.2663
    Elapsed Time =    0.27 Seconds

      profit
      i
      value
      resid
      j*
      value[close]
      value[open]

      profit

      idle
      -20.000000
      -20.000000
      0
      0.894342
      -1.110223e-16
      close
      0.894342
      -29.105658

      -19.983994
      -19.983994
      0
      0.894542
      -5.251344e-11
      close
      0.894542
      -29.089452

      -19.967987
      -19.967987
      0
      0.894743
      -5.991030e-11
      close
      0.894743
      -29.073245

      -19.951981
      -19.951981
      0
      0.894943
      -2.191858e-11
      close
      0.894943
      -29.057038

      -19.935974
      -19.935974
      0
      0.895144
      4.235901e-11
      close
      0.895144
      -29.040830

### Plot Action-Contingent Value Functions

```python
fig1,axs = plt.subplots(1,2,sharey=True, figsize=[12,5])
offs = [(4, -8), (-8, 5)]
msgs = ['Profit Entry', 'Profit Exit']

for i,iname in enumerate(dstates):
    plt.axes(axs[i])
    subdata = S.loc[iname,['value[close]', 'value[open]']]
    plt.plot(subdata)
    plt.title('Value of a currently ' + iname + ' firm')
    plt.xlabel('Potential Profit')
    if not i: plt.ylabel('Value of Firm')
    plt.legend(dactions)

    subdata['v.diff'] = subdata['value[open]'] - subdata['value[close]']
    pcrit = np.interp(0, subdata['v.diff'], subdata.index)
    vcrit  = np.interp(pcrit, subdata.index, subdata['value[close]'])
    demo.annotate(pcrit, vcrit, f'$p^*_{i:d} = {pcrit:.2f}$', 'wo', offs[i], fs=11, ms=12,)
    print(f'{msgs[i]:12s} = {pcrit:5.2f}')

```

### Plot Residual

We normalize the residuals as percentage of the value function. Notice the spikes at the "Profit entry" and "Profit exit" points.

```python
S['resid2'] = 100 * (S.resid / S.value)
fig2 = demo.figure('Bellman Equation Residual','Potential Profit','Percent Residual')
demo.qplot('profit','resid2','i',S)
plt.legend(dactions)
```

## SIMULATION

We simulate the model 50000 times for a time horizon $T=50$, starting with an operating firm ($d=1$) at the long-term profit mean $\bar{\pi}$. To be able to reproduce these results, we set the random seed at an arbitrary value of 945.

```python
T = 50
nrep = 50000
p0 = np.tile(pbar, (1, nrep))
d0 = 1
data = model.simulate(T, p0, d0, seed=945)
```

### Print Ergodic Moments

```python
f = '\t{:21s} = {:5.2f}'
print('\nErgodic Means')
print(f.format('Profit Contribution', data['profit'].mean()))
print(f.format('Activity', (data['i'] == 'active').mean()))
print('\nErgodic Standard Deviations\n')
print(f.format('Profit Contribution', data['profit'].std()))
print(f.format('Activity', (data['i'] == 'active').std()))
```

    Ergodic Means
    	Profit Contribution   =  1.00
    	Activity              =  0.94

    Ergodic Standard Deviations

    	Profit Contribution   =  1.37
    	Activity              =  0.24

```python
ergodic = pd.DataFrame({
    'Ergodic Means' : [data['profit'].mean(), (data['i'] == 'active').mean()],
    'Ergodic Standard Deviations': [data['profit'].std(), (data['i'] == 'active').std()]},
    index=['Profit Contribution', 'Activity'])

ergodic.round(2)
```

      Ergodic Means
      Ergodic Standard Deviations

      Profit Contribution
      1.00
      1.37

      Activity
      0.94
      0.24

### Plot Simulated and Expected Continuous State Path

```python
subdata = data[data['_rep'].isin(range(3))]

fig3 = demo.figure('Simulated and Expected Profit Contribution', 'Period','Profit Contribution')
demo.qplot('time','profit','_rep',subdata)
plt.plot(data[['time','profit']].groupby('time').mean(),'k--',label='mean')
plt.legend()
```

### Plot Expected Discrete State Path

```python
fig4 = demo.figure('Probability of Operation', 'Period', 'Probability')
data['ii'] = data['i'] == 'active'
plt.plot(data[['time','ii']].groupby('time').mean())
```

```python
#demo.savefig([fig1,fig2,fig3,fig4])
```

_build/html/_sources/notebooks/dp/03 Industry Entry-Exit Model.ipynb