```python
import tensorflow as tf
import tensorflow_probability as tfp
tfd = tfp.distributions

print("TF version:", tf.__version__)
print("TFP version:", tfp.__version__)
```

    TF version: 2.3.0
    TFP version: 0.11.0

```python
# Additional imports and setting fixed random seed to have reproducibility

import matplotlib.pyplot as plt
import numpy as np
tf.random.set_seed(123)
```

# Tensorflow Distributions

## Coding Tutorials
#### 1. [Univariate Distributions](#univariate_distributions)
#### 2. [Multivariate Distributions](#multivariate_distributions)
#### 3. [The Independent Distribution](#the_independent_distribution)
#### 4. [Sampling and log probs](#sampling_and_log_probs)
#### 5. [Trainable Distributions](#trainable_distributions)

***
## __a) Univariate distributions__

```python
# Create a normal distribution from Tensorflow Distributions

normal = tfd.Normal(loc=0, scale=1)
```

```python
# Sample from the chosen distribution...

normal.sample()
```

    <tf.Tensor: shape=(), dtype=float32, numpy=-0.8980837>

```python
# ... or sample multiple times

normal.sample(10)
```

    <tf.Tensor: shape=(10,), dtype=float32, numpy=
    array([ 0.33875433,  0.3449861 , -0.6605785 , -0.2854994 ,  0.43852386,
            0.8288566 , -0.53591555, -0.5353483 , -1.0324248 , -2.942705  ],
          dtype=float32)>

```python
# Obtain value of probability's density

normal.prob(0)
```

    <tf.Tensor: shape=(), dtype=float32, numpy=0.3989423>

```python
# Obtain value of logprobability

normal.log_prob(0)
```

    <tf.Tensor: shape=(), dtype=float32, numpy=-0.9189385>

```python
# Verify that this really is the log of the probability

np.log(normal.prob(0))
```

    -0.9189385

```python
# Plot a histogram, approximating the density

plt.hist(normal.sample(10000), bins=50, density=True)
plt.show()
```

```python
# Do the same for the exponential distribution

exponential = tfd.Exponential(rate=1)
plt.hist(exponential.sample(10000), bins=50, density=True)
plt.show()
```

```python
# Sample as before

exponential.sample(10)
```

    <tf.Tensor: shape=(10,), dtype=float32, numpy=
    array([0.3179616 , 0.9595855 , 0.6190708 , 1.3738598 , 1.6796893 ,
           0.40142855, 1.6830153 , 1.78942   , 0.38126466, 0.55283946],
          dtype=float32)>

```python
# Create a Bernoulli distribution (discrete)

bernoulli = tfd.Bernoulli(probs=0.8)
bernoulli.sample(20)
```

    <tf.Tensor: shape=(20,), dtype=int32, numpy=
    array([0, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 0, 1, 1],
          dtype=int32)>

#### A word of caution on discrete distributions

```python
# Calculate Bernoulli prob and see that 0.5 and -1 do not give the correct probability!

for k in [0,0.5,1,-1]:
    print('prob result {} for k = {} '.format(bernoulli.prob(k), k))
```

    prob result 0.19999998807907104 for k = 0
    prob result 0.3999999761581421 for k = 0.5
    prob result 0.800000011920929 for k = 1
    prob result 0.04999999701976776 for k = -1

```python
# Replicate the scores to see what is occurring under the hood

def my_bernoulli(p_sucess, k):
    return np.power(p_sucess,k)*np.power((1-p_sucess),(1-k))
```

```python
# Evaluate it as before

for k in [0,0.5,1,-1]:
    print('prob result {} for k = {} '.format(my_bernoulli(p_sucess=0.8, k=k), k))
```

    prob result 0.19999999999999996 for k = 0
    prob result 0.3999999999999999 for k = 0.5
    prob result 0.8 for k = 1
    prob result 0.049999999999999975 for k = -1

#### Work with batch distributions

```python
# Create a batched Bernoulli distribution

bernoulli_batch = tfd.Bernoulli(probs=[0.1, 0.25, 0.5, 0.75, 0.9])
bernoulli_batch
```

    <tfp.distributions.Bernoulli 'Bernoulli' batch_shape=[5] event_shape=[] dtype=int32>

```python
# Sample from it, noting the shape

bernoulli_batch.sample(5)
```

    <tf.Tensor: shape=(5, 5), dtype=int32, numpy=
    array([[0, 0, 0, 1, 1],
           [0, 1, 0, 1, 1],
           [0, 0, 1, 1, 1],
           [0, 0, 0, 1, 1],
           [0, 0, 0, 1, 1]], dtype=int32)>

```python
# Use a batch shape with higher rank

probs = [[[0.5, 0.5],
          [0.8, 0.3],
          [0.25, 0.75]]]
bernoulli_batch_2D = tfd.Bernoulli(probs=probs)
bernoulli_batch_2D
```

    <tfp.distributions.Bernoulli 'Bernoulli' batch_shape=[1, 3, 2] event_shape=[] dtype=int32>

```python
# Sample from this batch of distributions

bernoulli_batch_2D.sample(5)
```

    <tf.Tensor: shape=(5, 1, 3, 2), dtype=int32, numpy=
    array([[[[0, 0],
             [1, 1],
             [0, 1]]],

           [[[0, 0],
             [1, 0],
             [0, 1]]],

           [[[1, 0],
             [1, 1],
             [0, 1]]],

           [[[1, 0],
             [0, 0],
             [0, 1]]],

           [[[0, 1],
             [1, 1],
             [1, 0]]]], dtype=int32)>

```python
# Determine probabilities from this batch distribution

bernoulli_batch_2D.prob([[[1, 0],
                         [0, 0],
                         [1, 1]]])
```

    <tf.Tensor: shape=(1, 3, 2), dtype=float32, numpy=
    array([[[0.5       , 0.5       ],
            [0.19999999, 0.7       ],
            [0.25      , 0.75      ]]], dtype=float32)>

***

## __b) Multivariate Distributions__

#### Basic multivariate distributions

```python
# Define 2D multivariate Gaussian with diagonal covariance matrix

normal_diag = tfd.MultivariateNormalDiag(loc=[0, 1], scale_diag=[1, 2])  # scale_diag=[1, 20]) we can change the shape by altering the mean and SD
normal_diag
```

    <tfp.distributions.MultivariateNormalDiag 'MultivariateNormalDiag' batch_shape=[] event_shape=[2] dtype=float32>

```python
# Sample from it

normal_diag.sample(10)
```

    <tf.Tensor: shape=(10, 2), dtype=float32, numpy=
    array([[-0.37992278,  2.3674164 ],
           [-2.224005  , -0.28514445],
           [ 0.923083  , -1.4528892 ],
           [-0.62774605, -0.33852375],
           [-0.6252951 , -1.3324146 ],
           [-0.42454168,  1.3192186 ],
           [-1.702882  ,  1.8533869 ],
           [-0.4608376 , -0.7023523 ],
           [-1.1919353 , -0.12865627],
           [ 0.48053816, -0.2693485 ]], dtype=float32)>

```python
# Make a plot

plt_sample = normal_diag.sample(10000)
plt.scatter(plt_sample[:, 0], plt_sample[:, 1], marker='.', alpha=0.05)
plt.axis('equal')
plt.show()
```

#### Batches of multivariate distributions

```python
# Create three "batches" of multivariate normals

normal_diag_batch = tfd.MultivariateNormalDiag(loc=[[0, 0], [0, 0], [0, 0]],
                                               scale_diag=[[1, 2], [2, 1], [2, 2]])
normal_diag_batch
```

    <tfp.distributions.MultivariateNormalDiag 'MultivariateNormalDiag' batch_shape=[3] event_shape=[2] dtype=float32>

```python
# Sample from it

samples =normal_diag_batch.sample(5)
samples
```

    <tf.Tensor: shape=(5, 3, 2), dtype=float32, numpy=
    array([[[-0.8012545 , -2.128108  ],
            [ 2.0774972 , -2.7921855 ],
            [ 0.52665955,  0.6095758 ]],

           [[ 0.9923561 , -0.9778331 ],
            [-0.8376892 ,  0.70630246],
            [-1.0894657 , -0.65969497]],

           [[-1.6264789 ,  2.2429495 ],
            [-4.301875  , -0.7626804 ],
            [-0.43451956, -0.57022095]],

           [[ 0.73075646,  2.8356621 ],
            [ 1.8173586 ,  1.2079152 ],
            [-3.2939956 ,  2.33647   ]],

           [[-0.24759005,  0.5630618 ],
            [-0.6053428 ,  0.06578209],
            [ 1.4922864 , -0.55439734]]], dtype=float32)>

```python
# Compute log probs

normal_diag_batch.log_prob(samples)
```

    <tf.Tensor: shape=(5, 3), dtype=float32, numpy=
    array([[-3.4181342, -6.9686737, -3.3052905],
           [-3.142929 , -2.8681712, -3.426938 ],
           [-4.4825935, -5.1351314, -3.2884164],
           [-3.8031492, -3.6734028, -5.262859 ],
           [-2.6013045, -2.5789928, -3.5409558]], dtype=float32)>

```python
# Create a sample for a plot -- notice the shape

plt_sample_batch = normal_diag_batch.sample(10000)
plt_sample_batch.shape
```

    TensorShape([10000, 3, 2])

```python
# Plot samples from the batched multivariate Gaussian

fig, axs = (plt.subplots(1, 3, sharex=True, sharey=True, figsize=(10, 3)))
titles = ['cov_diag=[1, 2]','cov_diag=[2, 1]', 'cov_diag=[2, 2]']

for i, (ax, title) in enumerate(zip(axs,titles)):
    samples = plt_sample_batch[:,i,:] #take the ith batch [samples x event_shape]
    ax.scatter(samples[:, 0], samples[:, 1], marker='.', alpha=0.05)
    ax.set_title(title)
plt.show()
```

***

## __c) The Independent Distribution__

```python
# Start by defining a batch of two univariate Gaussians, then
# combine them into a bivariate Gaussian with independent components

locs = [-1., 1]
scales = [0.5, 1.]
batch_of_normals = tfd.Normal(loc=locs, scale=scales)
```

```python
# Univariate density functions

import seaborn as sns

t = np.linspace(-4, 4, 10000)
densities = batch_of_normals.prob(np.repeat(t[:, np.newaxis], 2, axis=1)) # each column is a vector of densities for one distn

sns.lineplot(t, densities[:, 0], label='loc={}, scale={}'.format(locs[0], scales[0]))
sns.lineplot(t, densities[:, 1], label='loc={}, scale={}'.format(locs[0], scales[0]))
plt.ylabel('Probability density')
plt.xlabel('Value')
plt.legend()
plt.show()
```

```python
# Check their batch_shape and event_shape

batch_of_normals
```

    <tfp.distributions.Normal 'Normal' batch_shape=[2] event_shape=[] dtype=float32>

```python
# Use Independent to convert the batch shape to the event shape

bivariate_normal_from_Independent = tfd.Independent(batch_of_normals,
                                                    reinterpreted_batch_ndims=1)
```

```python
# Note that dimension from batch_shape has shifted to event_shape

bivariate_normal_from_Independent
```

    <tfp.distributions.Independent 'IndependentNormal' batch_shape=[] event_shape=[2] dtype=float32>

```python
# Create a plot showing joint density contours and marginal density functions

samples = bivariate_normal_from_Independent.sample(10000)
x1 = samples[:, 0]
x2 = samples[:, 1]
sns.jointplot(x1, x2, kind="kde", space=0, color='b', xlim=[-4, 4], ylim=[-4, 4])
```

```python
# Use MultivariateNormalDiag to create the equivalent distribution
# Note that diagonal covariance matrix => no correlation => independence (for the multivariate normal distribution)

bivariate_normal_from_Multivariate = tfd.MultivariateNormalDiag(loc=locs,
                                                                scale_diag=scales)
```

```python
# Plot the joint density function of bivariate_normal_from_Independent
# Refer back to bivariate_normal_from_Independent to show that the plot is the same
# Summarise how Independent has been used

samples = bivariate_normal_from_Multivariate.sample(10000)
x1 = samples[:, 0]
x2 = samples[:, 1]
sns.jointplot(x1, x2, kind="kde", space=0, color='b', xlim=[-4, 4], ylim=[-4, 4])
```

#### Shifting batch dimensions to event dimensions using
`reinterpreted_batch_ndims`

```python
# Demonstrate use of reinterpreted_batch_ndims
# By default all batch dims except the first are transferred to event dims

loc_grid = [[-100., -100.],
            [100., 100.],
            [0., 0.]]
scale_grid = [[1., 10.],
              [1., 10.],
              [1., 1.]]

normals_batch_3by2_event_1 = tfd.Normal(loc=loc_grid, scale=scale_grid)
```

```python
# Highlight batch_shape

normals_batch_3by2_event_1
```

    <tfp.distributions.Normal 'Normal' batch_shape=[3, 2] event_shape=[] dtype=float32>

```python
# We now have a batch of 3 bivariate normal distributions,
# each parametrised by a column of our original parameter grid

normals_batch_3_event_2 =tfd.Independent(normals_batch_3by2_event_1)
normals_batch_3_event_2
```

    <tfp.distributions.Independent 'IndependentNormal' batch_shape=[3] event_shape=[2] dtype=float32>

```python
# Evaluate log_prob

normals_batch_3_event_2.log_prob(value=[[-10., 10,],
                                        [100., 100],
                                        [1., 1.]])
```

    <tf.Tensor: shape=(3,), dtype=float32, numpy=array([-4.1146406e+03, -4.1404624e+00, -2.8378770e+00], dtype=float32)>

```python
# Can reinterpret _all_ batch dimensions as event dimensions

normals_batch_1_event_3by2 = tfd.Independent(normals_batch_3by2_event_1,
                                             reinterpreted_batch_ndims=2)
normals_batch_1_event_3by2
```

    <tfp.distributions.Independent 'IndependentNormal' batch_shape=[] event_shape=[3, 2] dtype=float32>

```python
# Take log_probs

normals_batch_1_event_3by2.log_prob(value=[[-10., 10,],
                                           [100., 100],
                                           [1., 1.]])
```

    <tf.Tensor: shape=(), dtype=float32, numpy=-4121.619>

#### __Using `Independent` to build a Naive Bayes classifier__

#### Introduction to `newsgroups` data set

In this tutorial, just load the dataset, fetch train/test splits, probably choose a subset of the data.

Construct the class conditional feature distribution (with Independent, using the Naive Bayes assumption) and sample from it.

We can just use the ML estimates for parameters, in later tutorials we will learn them.

#### Import the data

The dataset required for this coding tutorial can be downloaded from the following link:

[20_Newsgroup_Data.zip](https://github.com/mohd-faizy/Probabilistic-Deep-Learning-with-TensorFlow/raw/main/01_The%20TensorFlow_Probability_library/20_Newsgroup_Data.zip)

```python
! wget https://github.com/mohd-faizy/Probabilistic-Deep-Learning-with-TensorFlow/raw/main/01_The%20TensorFlow_Probability_library/20_Newsgroup_Data.zip
```

    --2020-12-06 21:15:29--  https://github.com/mohd-faizy/Probabilistic-Deep-Learning-with-TensorFlow/raw/main/01_The%20TensorFlow_Probability_library/20_Newsgroup_Data.zip
    Resolving github.com (github.com)... 140.82.114.3
    Connecting to github.com (github.com)|140.82.114.3|:443... connected.
    HTTP request sent, awaiting response... 302 Found
    Location: https://raw.githubusercontent.com/mohd-faizy/Probabilistic-Deep-Learning-with-TensorFlow/main/01_The%20TensorFlow_Probability_library/20_Newsgroup_Data.zip [following]
    --2020-12-06 21:15:29--  https://raw.githubusercontent.com/mohd-faizy/Probabilistic-Deep-Learning-with-TensorFlow/main/01_The%20TensorFlow_Probability_library/20_Newsgroup_Data.zip
    Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.0.133, 151.101.64.133, 151.101.128.133, ...
    Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.0.133|:443... connected.
    HTTP request sent, awaiting response... 200 OK
    Length: 15345225 (15M) [application/zip]
    Saving to: ‘20_Newsgroup_Data.zip’

    20_Newsgroup_Data.z 100%[===================>]  14.63M  38.6MB/s    in 0.4s

    2020-12-06 21:15:30 (38.6 MB/s) - ‘20_Newsgroup_Data.zip’ saved [15345225/15345225]

```python
! unzip -uq '/content/20_Newsgroup_Data.zip' -d '/content'
```

```python
# Convenience function for retrieving the 20 newsgroups data set

# Usenet was a forerunner to modern internet forums
# Users could post and read articles
# Newsgroup corresponded to a topic
# Example topics in this data set: IBM computer hardware, baseball
# Our objective is to use an article's contents to predict its newsgroup,
# a 20-class classification problem.

# 18000 newsgroups, posts on 20 topics
from sklearn.datasets import fetch_20newsgroups
from sklearn.feature_extraction.text import CountVectorizer

```

```python
# Get the train data

newsgroups_data = fetch_20newsgroups(data_home='/content/20_Newsgroup_Data', subset='train')
```

```python
# More information about the data set

print(newsgroups_data['DESCR'])
```

    .. _20newsgroups_dataset:

    The 20 newsgroups text dataset
    ------------------------------

    The 20 newsgroups dataset comprises around 18000 newsgroups posts on
    20 topics split in two subsets: one for training (or development)
    and the other one for testing (or for performance evaluation). The split
    between the train and test set is based upon a messages posted before
    and after a specific date.

    This module contains two loaders. The first one,
    :func:`sklearn.datasets.fetch_20newsgroups`,
    returns a list of the raw texts that can be fed to text feature
    extractors such as :class:`sklearn.feature_extraction.text.CountVectorizer`
    with custom parameters so as to extract feature vectors.
    The second one, :func:`sklearn.datasets.fetch_20newsgroups_vectorized`,
    returns ready-to-use features, i.e., it is not necessary to use a feature
    extractor.

    **Data Set Characteristics:**

        =================   ==========
        Classes                     20
        Samples total            18846
        Dimensionality               1
        Features                  text
        =================   ==========

    Usage
    ~~~~~

    The :func:`sklearn.datasets.fetch_20newsgroups` function is a data
    fetching / caching functions that downloads the data archive from
    the original `20 newsgroups website`_, extracts the archive contents
    in the ``~/scikit_learn_data/20news_home`` folder and calls the
    :func:`sklearn.datasets.load_files` on either the training or
    testing set folder, or both of them::

      >>> from sklearn.datasets import fetch_20newsgroups
      >>> newsgroups_train = fetch_20newsgroups(subset='train')

      >>> from pprint import pprint
      >>> pprint(list(newsgroups_train.target_names))
      ['alt.atheism',
       'comp.graphics',
       'comp.os.ms-windows.misc',
       'comp.sys.ibm.pc.hardware',
       'comp.sys.mac.hardware',
       'comp.windows.x',
       'misc.forsale',
       'rec.autos',
       'rec.motorcycles',
       'rec.sport.baseball',
       'rec.sport.hockey',
       'sci.crypt',
       'sci.electronics',
       'sci.med',
       'sci.space',
       'soc.religion.christian',
       'talk.politics.guns',
       'talk.politics.mideast',
       'talk.politics.misc',
       'talk.religion.misc']

    The real data lies in the ``filenames`` and ``target`` attributes. The target
    attribute is the integer index of the category::

      >>> newsgroups_train.filenames.shape
      (11314,)
      >>> newsgroups_train.target.shape
      (11314,)
      >>> newsgroups_train.target[:10]
      array([ 7,  4,  4,  1, 14, 16, 13,  3,  2,  4])

    It is possible to load only a sub-selection of the categories by passing the
    list of the categories to load to the
    :func:`sklearn.datasets.fetch_20newsgroups` function::

      >>> cats = ['alt.atheism', 'sci.space']
      >>> newsgroups_train = fetch_20newsgroups(subset='train', categories=cats)

      >>> list(newsgroups_train.target_names)
      ['alt.atheism', 'sci.space']
      >>> newsgroups_train.filenames.shape
      (1073,)
      >>> newsgroups_train.target.shape
      (1073,)
      >>> newsgroups_train.target[:10]
      array([0, 1, 1, 1, 0, 1, 1, 0, 0, 0])

    Converting text to vectors
    ~~~~~~~~~~~~~~~~~~~~~~~~~~

    In order to feed predictive or clustering models with the text data,
    one first need to turn the text into vectors of numerical values suitable
    for statistical analysis. This can be achieved with the utilities of the
    ``sklearn.feature_extraction.text`` as demonstrated in the following
    example that extract `TF-IDF`_ vectors of unigram tokens
    from a subset of 20news::

      >>> from sklearn.feature_extraction.text import TfidfVectorizer
      >>> categories = ['alt.atheism', 'talk.religion.misc',
      ...               'comp.graphics', 'sci.space']
      >>> newsgroups_train = fetch_20newsgroups(subset='train',
      ...                                       categories=categories)
      >>> vectorizer = TfidfVectorizer()
      >>> vectors = vectorizer.fit_transform(newsgroups_train.data)
      >>> vectors.shape
      (2034, 34118)

    The extracted TF-IDF vectors are very sparse, with an average of 159 non-zero
    components by sample in a more than 30000-dimensional space
    (less than .5% non-zero features)::

      >>> vectors.nnz / float(vectors.shape[0])
      159.01327...

    :func:`sklearn.datasets.fetch_20newsgroups_vectorized` is a function which
    returns ready-to-use token counts features instead of file names.

    .. _`20 newsgroups website`: http://people.csail.mit.edu/jrennie/20Newsgroups/
    .. _`TF-IDF`: https://en.wikipedia.org/wiki/Tf-idf

    Filtering text for more realistic training
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

    It is easy for a classifier to overfit on particular things that appear in the
    20 Newsgroups data, such as newsgroup headers. Many classifiers achieve very
    high F-scores, but their results would not generalize to other documents that
    aren't from this window of time.

    For example, let's look at the results of a multinomial Naive Bayes classifier,
    which is fast to train and achieves a decent F-score::

      >>> from sklearn.naive_bayes import MultinomialNB
      >>> from sklearn import metrics
      >>> newsgroups_test = fetch_20newsgroups(subset='test',
      ...                                      categories=categories)
      >>> vectors_test = vectorizer.transform(newsgroups_test.data)
      >>> clf = MultinomialNB(alpha=.01)
      >>> clf.fit(vectors, newsgroups_train.target)
      MultinomialNB(alpha=0.01, class_prior=None, fit_prior=True)

      >>> pred = clf.predict(vectors_test)
      >>> metrics.f1_score(newsgroups_test.target, pred, average='macro')
      0.88213...

    (The example :ref:`sphx_glr_auto_examples_text_plot_document_classification_20newsgroups.py` shuffles
    the training and test data, instead of segmenting by time, and in that case
    multinomial Naive Bayes gets a much higher F-score of 0.88. Are you suspicious
    yet of what's going on inside this classifier?)

    Let's take a look at what the most informative features are:

      >>> import numpy as np
      >>> def show_top10(classifier, vectorizer, categories):
      ...     feature_names = np.asarray(vectorizer.get_feature_names())
      ...     for i, category in enumerate(categories):
      ...         top10 = np.argsort(classifier.coef_[i])[-10:]
      ...         print("%s: %s" % (category, " ".join(feature_names[top10])))
      ...
      >>> show_top10(clf, vectorizer, newsgroups_train.target_names)
      alt.atheism: edu it and in you that is of to the
      comp.graphics: edu in graphics it is for and of to the
      sci.space: edu it that is in and space to of the
      talk.religion.misc: not it you in is that and to of the

    You can now see many things that these features have overfit to:

    - Almost every group is distinguished by whether headers such as
      ``NNTP-Posting-Host:`` and ``Distribution:`` appear more or less often.
    - Another significant feature involves whether the sender is affiliated with
      a university, as indicated either by their headers or their signature.
    - The word "article" is a significant feature, based on how often people quote
      previous posts like this: "In article [article ID], [name] <[e-mail address]>
      wrote:"
    - Other features match the names and e-mail addresses of particular people who
      were posting at the time.

    With such an abundance of clues that distinguish newsgroups, the classifiers
    barely have to identify topics from text at all, and they all perform at the
    same high level.

    For this reason, the functions that load 20 Newsgroups data provide a
    parameter called **remove**, telling it what kinds of information to strip out
    of each file. **remove** should be a tuple containing any subset of
    ``('headers', 'footers', 'quotes')``, telling it to remove headers, signature
    blocks, and quotation blocks respectively.

      >>> newsgroups_test = fetch_20newsgroups(subset='test',
      ...                                      remove=('headers', 'footers', 'quotes'),
      ...                                      categories=categories)
      >>> vectors_test = vectorizer.transform(newsgroups_test.data)
      >>> pred = clf.predict(vectors_test)
      >>> metrics.f1_score(pred, newsgroups_test.target, average='macro')
      0.77310...

    This classifier lost over a lot of its F-score, just because we removed
    metadata that has little to do with topic classification.
    It loses even more if we also strip this metadata from the training data:

      >>> newsgroups_train = fetch_20newsgroups(subset='train',
      ...                                       remove=('headers', 'footers', 'quotes'),
      ...                                       categories=categories)
      >>> vectors = vectorizer.fit_transform(newsgroups_train.data)
      >>> clf = MultinomialNB(alpha=.01)
      >>> clf.fit(vectors, newsgroups_train.target)
      MultinomialNB(alpha=0.01, class_prior=None, fit_prior=True)

      >>> vectors_test = vectorizer.transform(newsgroups_test.data)
      >>> pred = clf.predict(vectors_test)
      >>> metrics.f1_score(newsgroups_test.target, pred, average='macro')
      0.76995...

    Some other classifiers cope better with this harder version of the task. Try
    running :ref:`sphx_glr_auto_examples_model_selection_grid_search_text_feature_extraction.py` with and without
    the ``--filter`` option to compare the results.

    .. topic:: Recommendation

      When evaluating text classifiers on the 20 Newsgroups data, you
      should strip newsgroup-related metadata. In scikit-learn, you can do this by
      setting ``remove=('headers', 'footers', 'quotes')``. The F-score will be
      lower because it is more realistic.

    .. topic:: Examples

       * :ref:`sphx_glr_auto_examples_model_selection_grid_search_text_feature_extraction.py`

       * :ref:`sphx_glr_auto_examples_text_plot_document_classification_20newsgroups.py`

```python
# Example article

print(newsgroups_data['data'][0])
```

    From: lerxst@wam.umd.edu (where's my thing)
    Subject: WHAT car is this!?
    Nntp-Posting-Host: rac3.wam.umd.edu
    Organization: University of Maryland, College Park
    Lines: 15

     I was wondering if anyone out there could enlighten me on this car I saw
    the other day. It was a 2-door sports car, looked to be from the late 60s/
    early 70s. It was called a Bricklin. The doors were really small. In addition,
    the front bumper was separate from the rest of the body. This is
    all I know. If anyone can tellme a model name, engine specs, years
    of production, where this car is made, history, or whatever info you
    have on this funky looking car, please e-mail.

    Thanks,
    - IL
       ---- brought to you by your neighborhood Lerxst ----

```python
# Associated label

newsgroups_data['target'][0]
```

    7

```python
# Name of label

newsgroups_data['target_names'][7]
```

    'rec.autos'

```python
# Preprocessing boilerplate

n_documents = len(newsgroups_data['data'])

count_vectorizer = CountVectorizer(input='content', binary=True,
                                   max_df=0.25, min_df=1.01/n_documents) # ignore common words, words that appear once
binary_bag_of_words = count_vectorizer.fit_transform(newsgroups_data['data']) # input is a list of strings

```

```python
# Check shape

binary_bag_of_words.shape
```

    (11314, 56365)

```python
# Check that the fit has been successful

count_vectorizer.inverse_transform(binary_bag_of_words[0, :])
```

    [array(['lerxst', 'wam', 'umd', 'where', 'thing', 'car', 'rac3',
            'maryland', 'college', 'park', '15', 'wondering', 'anyone',
            'could', 'enlighten', 'saw', 'day', 'door', 'sports', 'looked',
            'late', '60s', 'early', '70s', 'called', 'bricklin', 'doors',
            'were', 'really', 'small', 'addition', 'front', 'bumper',
            'separate', 'rest', 'body', 'tellme', 'model', 'name', 'engine',
            'specs', 'years', 'production', 'made', 'history', 'whatever',
            'info', 'funky', 'looking', 'please', 'mail', 'thanks', 'il',
            'brought', 'neighborhood'], dtype='<U80')]

```python
# Dict that will be useful later

inv_vocabulary = {value:key for key, value in count_vectorizer.vocabulary_.items()}
```

#### A Naive Bayes classifier for `newsgroup`

Each feature vector $x$ is a list of indicators for whether a word appears in the article. $x_i$ is 1 if the $i$th word appears, and 0 otherwise. `inv_vocabulary` matches word indices $i$ to words.

Each label $y$ is a value in $0, 1, \ldots, 19$.

The parts of a naive Bayes classifier for this problem can be summarised as:

- A probability distribution for the feature vector by class, $p(x|y = j)$ for each $j = 0, 1, \ldots, 19$. These probability distributions are assumed to have independent components: we can factorize the joint probability as a product of marginal probabilities
\begin{equation}
    p(x|y = j) = \prod_{i=1}^d p(x_i|y = j)
\end{equation}
These marginal probability distributions are Bernoulli distributions, each of which has a single parameter $\theta_{ji} := p(x_i = 1|y = j)$. This parameter is the probability of observing word $i$ in an article of class $j$.

- We will use the Laplace smoothed maximum likelihood estimate to compute these parameters. Laplace smoothing involves adding small counts to every feature for each class. Else, if a feature did not appear in the training set of a class, but then we observed it in our test data the log probability would be undefined.

- A collection of class prior probabilities $p(y = j)$. These will be set by computing the class base rates in the training set.

- A function for computing the probability of class membership via Bayes' theorem:

\begin{equation}
    p(y = j|x) = \frac{p(x|y = j)p(y = j)}{p(x)}
\end{equation}

```python
# Compute the parameter estimates (adjusted fraction of documents in class that contain word)

n_classes = newsgroups_data['target'].max() + 1
y = newsgroups_data['target']
n_words = binary_bag_of_words.shape[1]

alpha = 1e-6 # parameters for Laplace smoothing

theta = np.zeros([n_classes, n_words]) # stores parameter values - prob. word given class
for c_k in range(n_classes): # 0, 1, ..., 19
    class_mask = (y == c_k)
    N = class_mask.sum() # number of articles in class
    theta[c_k, :] = (binary_bag_of_words[class_mask, :].sum(axis=0) + alpha)/(N + alpha*2)
```

```python
# Check whether the most probable word in each class is reasonable

most_probable_word_ix = theta.argmax(axis=1) # most probable word for each class

for j, ix in enumerate(most_probable_word_ix):
    print('Most probable word in class {} is "{}".'.format(newsgroups_data['target_names'][j],
                                                           inv_vocabulary[ix]))
```

    Most probable word in class alt.atheism is "people".
    Most probable word in class comp.graphics is "graphics".
    Most probable word in class comp.os.ms-windows.misc is "windows".
    Most probable word in class comp.sys.ibm.pc.hardware is "thanks".
    Most probable word in class comp.sys.mac.hardware is "mac".
    Most probable word in class comp.windows.x is "window".
    Most probable word in class misc.forsale is "sale".
    Most probable word in class rec.autos is "car".
    Most probable word in class rec.motorcycles is "dod".
    Most probable word in class rec.sport.baseball is "he".
    Most probable word in class rec.sport.hockey is "ca".
    Most probable word in class sci.crypt is "clipper".
    Most probable word in class sci.electronics is "use".
    Most probable word in class sci.med is "reply".
    Most probable word in class sci.space is "space".
    Most probable word in class soc.religion.christian is "god".
    Most probable word in class talk.politics.guns is "people".
    Most probable word in class talk.politics.mideast is "people".
    Most probable word in class talk.politics.misc is "people".
    Most probable word in class talk.religion.misc is "he".

```python
# Define a distribution for each class

batch_of_bernoullis = tfd.Bernoulli(probs=theta)
p_x_given_y = tfd.Independent(batch_of_bernoullis, reinterpreted_batch_ndims=1)
p_x_given_y
```

    <tfp.distributions.Independent 'IndependentBernoulli' batch_shape=[20] event_shape=[56365] dtype=int32>

```python
# Take a sample of words from each class

n_samples = 10
sample = p_x_given_y.sample(n_samples)
sample.shape
```

    TensorShape([10, 20, 56365])

```python
# Choose a class

chosen_class = 15
newsgroups_data['target_names'][chosen_class]
```

    'soc.religion.christian'

```python
# Indicators for words that appear in the sample

class_sample = sample[:, chosen_class, :]
class_sample
```

    <tf.Tensor: shape=(10, 56365), dtype=int32, numpy=
    array([[0, 0, 0, ..., 0, 0, 0],
           [0, 0, 0, ..., 0, 0, 0],
           [0, 0, 0, ..., 0, 0, 0],
           ...,
           [0, 0, 0, ..., 0, 0, 0],
           [0, 0, 0, ..., 0, 0, 0],
           [0, 0, 0, ..., 0, 0, 0]], dtype=int32)>

```python
# Perform inverse transform to test quality of fit

count_vectorizer.inverse_transform(class_sample)
```

    [array(['09', '19', '23', '28', '31', '34', '35', '41', '78228', 'acs',
            'actually', 'advise', 'ages', 'also', 'andrew', 'anyone',
            'anything', 'appease', 'apr', 'bathroom', 'bc', 'because',
            'beggar', 'believe', 'better', 'bigelow', 'brains', 'chest',
            'child', 'claim', 'cleaned', 'commented', 'concepts', 'country',
            'day', 'daycare', 'death', 'decided', 'definition', 'detailed',
            'difference', 'disciple', 'discuss', 'does', 'emotional',
            'encourage', 'even', 'excellent', 'explain', 'faith', 'familiar',
            'fellowship', 'forgiven', 'free', 'garnet', 'gentiles', 'go',
            'god', 'good', 'growing', 'guess', 'hear', 'heard', 'here', 'him',
            'hold', 'honoring', 'however', 'ignore', 'kind', 'knowing', 'last',
            'lawrence', 'least', 'legal', 'less', 'life', 'minds', 'moment',
            'monotheism', 'must', 'never', 'now', 'offers', 'original', 'our',
            'over', 'pa', 'path', 'perception', 'person', 'personally', 'phil',
            'political', 'predestination', 'previously', 'problem', 'problems',
            'prove', 'punish', 'questionnaire', 'qui', 'quite', 'quote',
            'quotes', 'reading', 'real', 'reasonable', 'requirement', 'result',
            'reversed', 'say', 'second', 'sense', 'shaken', 'small', 'sons',
            'souls', 'span', 'speaking', 'spiritually', 'ssd', 'statements',
            'stem', 'still', 'study', 'take', 'talk', 'testament', 'than',
            'their', 'them', 'then', 'theories', 'these', 'things', 'think',
            'through', 'trying', 'two', 'uk', 'understand', 'us', 'uu4',
            'verse', 'very', 'views', 'vituperousness', 'wasn', 'watt', 'way',
            'well', 'whose', 'why', 'willing', 'wisdom', 'words', 'world'],
           dtype='<U80'),
     array(['28379', '_is_', 'aaron', 'agree', 'already', 'am', 'another',
            'anullments', 'aptly', 'arrogance', 'atheist', 'attributes',
            'because', 'belief', 'benefits', 'boring', 'both', 'ca',
            'carefully', 'christian', 'christianity', 'come', 'comes',
            'compensate', 'course', 'covington', 'daily', 'depends',
            'different', 'disagreement', 'done', 'effect', 'effects',
            'embarrassed', 'error', 'everyone', 'exactly', 'f16', 'faith',
            'far', 'fax', 'find', 'fires', 'found', 'good', 'gospels', 'gov',
            'hard', 'harry', 'him', 'his', 'hope', 'imagined', 'into', 'jesus',
            'job', 'johnson', 'knew', 'live', 'll', 'made', 'make', 'many',
            'mark', 'may', 'means', 'mothers', 'motivate', 'myself',
            'mysteries', 'newsgroup', 'nor', 'odd', 'oh', 'omniscience',
            'ought', 'overrepresented', 'pardon', 'passed', 'philosophy',
            'place', 'possibly', 'prayers', 'pretty', 'psi', 'purpose',
            'question', 'random', 'rather', 'resolve', 'revelation', 'rights',
            'rock', 'rutgers', 'saint', 'same', 'senses', 'sides', 'speaking',
            'steve', 'such', 'take', 'taught', 'than', 'their', 'then',
            'things', 'translation', 'tried', 'true', 'turns', 'two', 'until',
            'using', 've', 'waco', 'want', 'way', 'whether', 'why', 'word',
            'wrong', 'wrongs', 'yahweh', 'yourselves'], dtype='<U80'),
     array(['03', '34', '_is_', '_source_', 'advanced', 'agree', 'am',
            'andrew', 'appropriate', 'april', 'argue', 'arrogance', 'athos',
            'authorities', 'aziz', 'become', 'before', 'believe', 'books',
            'calendar', 'called', 'case', 'certain', 'certainly', 'clearer',
            'comes', 'contradicts', 'covering', 'darius', 'davem', 'delivered',
            'derived', 'details', 'developments', 'did', 'die', 'different',
            'diverge', 'done', 'engineering', 'espionage', 'even', 'explain',
            'fate', 'filled', 'find', 'geneva', 'gregory', 'groping', 'hcx1',
            'healy', 'heart', 'here', 'him', 'himself', 'history', 'human',
            'intimidating', 'into', 'iscs', 'isn', 'jenk', 'jesus', 'jewish',
            'judging', 'keep', 'knowing', 'last', 'let', 'life', 'lying',
            'mail', 'making', 'ministry', 'most', 'natural', 'needs', 'never',
            'now', 'observations', 'offer', 'ohio', 'pacbell', 'people',
            'praxeas', 'praying', 'ps', 'r2d2', 'really', 'refused',
            'religion', 'religious', 'reply', 'reprinting', 'require', 'right',
            'rights', 'roboticist', 'said', 'san', 'save', 'say',
            'shakespeare', 'sharon', 'should', 'sin', 'skidmore', 'soon',
            'soul', 'sun', 'talks', 'testifies', 'thank', 'through', 'thus',
            'tim', 'trust', 'uk', 'until', 'vbv', 'want', 'wasn', 'watch',
            'without', 'witness', 'world', 'yet'], dtype='<U80'),
     array(['08', '12', '16', '1989', '50', '57', 'accomplish', 'according',
            'advocate', 'after', 'aisun3', 'allow', 'answers', 'anyway',
            'archbishop', 'before', 'beginning', 'believed', 'best', 'blessed',
            'both', 'century', 'child', 'christ', 'christians', 'claim',
            'claiming', 'come', 'common', 'compared', 'complete',
            'confucianism', 'conjunction', 'could', 'damned', 'dean', 'did',
            'difficulty', 'does', 'doing', 'down', 'enabling', 'equivalence',
            'even', 'everything', 'evidence', 'excellent', 'eyes', 'find',
            'full', 'gboro', 'god', 'grad', 'guild', 'had', 'hand', 'hatred',
            'having', 'he', 'heaven', 'heroic', 'however', 'hulman', 'idea',
            'important', 'impose', 'incarnate', 'inside', 'inspired',
            'instead', 'jeesus', 'jesus', 'john', 'kille', 'kingston', 'later',
            'laws', 'made', 'may', 'mdw33310', 'meet', 'minorities', 'most',
            'moved', 'name', 'need', 'new', 'niblick', 'nicea', 'occurred',
            'often', 'our', 'own', 'participants', 'passion', 'performed',
            'personal', 'physical', 'planet', 'probably', 'punishment',
            'questioning', 'quotes', 'regarded', 'reid', 'run', 'sabbath',
            'same', 'seem', 'seems', 'set', 'shares', 'sloth', 'smith',
            'solve', 'something', 'specially', 'strict', 'sufficient',
            'superset', 'take', 'taken', 'then', 'things', 'think', 'those',
            'told', 'tongue', 'top', 'translators', 'try', 'trying', 'ucs',
            'ucsd', 'upon', 'view', 'want', 'ways', 'well', 'were', 'while',
            'wise', 'without', 'wittgenstein', 'worship'], dtype='<U80'),
     array(['03', '08', '11', '122', '15', '17790', '19', '245', '823',
            'ability', 'accorded', 'acns', 'ad', 'against', 'also', 'am',
            'andrew', 'apr', 'argument', 'ashley', 'assuming', 'athos',
            'attention', 'away', 'because', 'been', 'before', 'belief',
            'beyond', 'bible', 'books', 'born', 'burnaby', 'capable',
            'capacity', 'christ', 'christian', 'christians', 'claim',
            'collect', 'college', 'could', 'cover', 'cross', 'crowd', 'cunyvm',
            'dead', 'dec', 'delivery', 'deserving', 'difference', 'down',
            'effect', 'energy', 'entities', 'equivalent', 'exist',
            'explanation', 'facilities', 'far', 'few', 'files', 'firstly',
            'follows', 'fundamental', 'garden', 'gets', 'god', 'goes', 'got',
            'great', 'grew', 'hale', 'hard', 'he', 'hi', 'him', 'his',
            'imitate', 'important', 'incredibly', 'indiana', 'into',
            'involved', 'jacob', 'jason', 'john', 'kicking', 'knowing',
            'knowledge', 'list', 'look', 'made', 'mail', 'maintaining', 'man',
            'mcovingt', 'michael', 'middle', 'mot', 'necessary', 'need', 'new',
            'note', 'ohio', 'origins', 'paste', 'perennial', 'please', 'poem',
            'possibility', 'presupposes', 'pride', 'question', 'qui', 'reason',
            'references', 'rejected', 'reliance', 'response', 'rise',
            'rubrics', 'rutgers', 'sarcasm', 'says', 'second', 'see', 'shake',
            'should', 'soon', 'specifically', 'spinning', 'spiritual',
            'stable', 'stands', 'state', 'statement', 'suppose', 'sure',
            'taber', 'take', 'ten', 'their', 'these', 'thought', 'time', 'too',
            'try', 'under', 'understanding', 'username', 'vice', 'want',
            'warped', 'welcome', 'were', 'whether', 'whole', 'work'],
           dtype='<U80'),
     array(['09', '10', '1835', '1993', '23', '29', '60', '825', '92', '9760',
            'able', 'abstinence', 'accept', 'activists', 'agron', 'also',
            'anyone', 'apr', 'asked', 'astray', 'because', 'being', 'believe',
            'believers', 'bgu', 'bible', 'blindness', 'bob', 'bound',
            'building', 'called', 'came', 'canaan', 'christian', 'christians',
            'city', 'concern', 'controlled', 'correctly', 'dan', 'danger',
            'dean', 'decided', 'descriptions', 'did', 'directions', 'dlecoint',
            'doing', 'ending', 'everyone', 'exchanged', 'existence',
            'expresses', 'feels', 'fnal', 'founder', 'fully', 'gained',
            'garnet', 'genesis', 'genocide', 'georgia', 'given', 'goodbye',
            'had', 'hard', 'he', 'him', 'his', 'jemurray', 'jesus', 'johnson',
            'joseph', 'kingdom', 'labs', 'lexington', 'line', 'lord', 'matter',
            'mean', 'meaning', 'message', 'met', 'motives', 'must', 'name',
            'nothing', 'nyu', 'often', 'our', 'perhaps', 'pithy', 'precious',
            'precise', 'priests', 'prism', 'prophet', 'purpose', 'put',
            'rather', 'resist', 'resurrection', 'rise', 'risk', 'ruthless',
            'said', 'say', 'says', 'school', 'search', 'should', 'simply',
            'simulation', 'skipper', 'small', 'state', 'statements', 'steven',
            'suffering', 'sure', 'talked', 'text', 'them', 'themselves',
            'theories', 'time', 'too', 'trinity', 'uga', 'uk', 'uribe', 'use',
            'used', 'useful', 'weren', 'wheaton', 'where', 'why', 'wrote',
            'yourself'], dtype='<U80'),
     array(['16', '25', '2a', '_want_', 'actions', 'actually', 'appropriate',
            'apr', 'arabs', 'ask', 'been', 'beings', 'big', 'bodies', 'book',
            'both', 'bothers', 'boundaries', 'brother', 'ca', 'certainly',
            'check', 'christ', 'christianity', 'christians', 'church', 'cia',
            'claim', 'classically', 'come', 'concern', 'consultant', 'courant',
            'current', 'department', 'didn', 'does', 'doesn', 'doing', 'dont',
            'enclosed', 'essence', 'essential', 'ethics', 'everyday', 'faith',
            'faiths', 'father', 'filled', 'first', 'forgetting', 'go',
            'grabbed', 'grace', 'graduate', 'great', 'hair', 'hard', 'he',
            'help', 'here', 'him', 'his', 'hold', 'hopper', 'identify',
            'intended', 'intent', 'invention', 'jewish', 'known', 'leave',
            'll', 'llo', 'make', 'marriages', 'matthew', 'may', 'must',
            'netters', 'never', 'new', 'normal', 'now', 'nt', 'oh',
            'oppressing', 'outside', 'party', 'peace', 'perfect', 'quite',
            'quoted', 'rabbi', 'read', 'real', 'religions', 'rely', 'repent',
            'respect', 'rutgers', 'sabbath', 'same', 'scriptura', 'send',
            'shall', 'should', 'something', 'speed', 'st', 'storrs',
            'successful', 'such', 'suppose', 'swear', 'than', 'them',
            'theologically', 'these', 'those', 'thrust', 'time', 'title',
            'truly', 'uchicago', 'ucsd', 'unfortunately', 'vey', 'wanted',
            'well', 'whole', 'whom', 'work', 'world', 'wrote'], dtype='<U80'),
     array(['01', '12', '13', '16', '66', 'able', 'after', 'almost', 'always',
            'amen', 'another', 'around', 'aside', 'attack', 'attacking',
            'attitudes', 'back', 'badly', 'before', 'being', 'believe',
            'better', 'bible', 'biblical', 'bjorn', 'body', 'book', 'books',
            'both', 'broken', 'bust', 'caralv', 'carl', 'catholic', 'change',
            'coming', 'constantly', 'contrary', 'could', 'cover', 'crazy',
            'crucify', 'deleted', 'desiree_bradley', 'destruction', 'discover',
            'disparate', 'draw', 'du', 'empire', 'especially', 'eternal',
            'everything', 'expressions', 'fact', 'find', 'first', 'founded',
            'georgia', 'give', 'god', 'halifax', 'having', 'him', 'his',
            'holy', 'horrible', 'however', 'idea', 'implications', 'important',
            'info', 'intensive', 'into', 'invited', 'its', 'jesus', 'jono',
            'judged', 'last', 'leviticus', 'lie', 'logic', 'longer', 'look',
            'mark', 'mentioned', 'michael', 'midway', 'mot', 'music', 'need',
            'net', 'next', 'non', 'nothing', 'often', 'opportunity', 'part',
            'participation', 'paul', 'people', 'perhaps', 'please', 'possible',
            'problem', 'quite', 'read', 'realize', 'really', 'reason',
            'relationship', 'relevant', 'reply', 'rest', 'rjs2', 'sacrifice',
            'same', 'satisfy', 'scientist', 'section', 'seen', 'sexually',
            'share', 'sole', 'something', 'soup', 'spend', 'state', 'sunday',
            'suspicion', 'their', 'through', 'today', 'took', 'toungues',
            'trivial', 'under', 'understand', 'unisys', 'usenet', 've', 'were',
            'whatever', 'whether', 'whosoever', 'wilkerson', 'without', 'word',
            'wrong', 'years'], dtype='<U80'),
     array(['01', '10', '110', '1349', '25', '36', 'above', 'achieve', 'acs',
            'afraid', 'andrew', 'anger', 'argument', 'arose', 'athens', 'away',
            'basic', 'bear', 'beard', 'because', 'been', 'being', 'bible',
            'came', 'carcs1', 'case', 'catholic', 'cbnewsk', 'cheap', 'church',
            'claim', 'colorado', 'come', 'comes', 'commit', 'conquer',
            'conscience', 'covington', 'cowardice', 'cruelty', 'curious',
            'davidian', 'dealt', 'dean', 'deficiencies', 'did', 'difference',
            'disproof', 'divisions', 'doing', 'drawer', 'dunedin', 'east',
            'effectively', 'either', 'eternity', 'even', 'ever', 'express',
            'faith', 'fax', 'few', 'figuring', 'find', 'fold', 'further',
            'germanic', 'give', 'god', 'gospel', 'happy', 'hebrews', 'hold',
            'honest', 'honor', 'however', 'idea', 'instead', 'involved',
            'jesus', 'keys', 'king', 'lay', 'life', 'lived', 'lives', 'll',
            'magazine', 'may', 'mean', 'messenger', 'motivate', 'never', 'new',
            'north', 'nunez', 'nyc', 'off', 'org', 'particular', 'permission',
            'points', 'possible', 'priority', 'reagan', 'really',
            'relationship', 'reply', 'require', 'retribution', 'revulsion',
            'right', 'rob', 'rome', 'said', 'say', 'scream', 'seems', 'single',
            'square', 'starting', 'straightforward', 'such', 'suffering',
            'support', 'talks', 'tape', 'than', 'think', 'until', 'values',
            'vela', 'verses', 'very', 'views', 'were', 'while', 'whuzzat',
            'wording', 'work', 'world'], dtype='<U80'),
     array(['11', '17', '1993', '54', 'above', 'accidents', 'according',
            'agonizing', 'aisun3', 'alive', 'american', 'among', 'andrew',
            'anger', 'asking', 'associated', 'ban', 'because', 'being',
            'benefit', 'bible', 'both', 'bring', 'cards', 'case', 'ceremony',
            'certainly', 'chrisitan', 'christian', 'comments', 'condemnation',
            'condemns', 'consign', 'copper', 'cosmic', 'cows', 'dead',
            'delivered', 'develop', 'ecn', 'etc', 'eventhough', 'evolution',
            'first', 'gets', 'go', 'going', 'got', 'grey', 'happening',
            'himself', 'his', 'hmmm', 'hold', 'husband', 'ignore', 'imply',
            'interior', 'interprets', 'involve', 'involved', 'jill',
            'kulikauskas', 'live', 'local', 'made', 'main', 'man', 'matthew',
            'mature', 'may', 'mean', 'meaning', 'mmalt', 'moderator',
            'mormonism', 'nationalism', 'network', 'non', 'numb', 'odder',
            'once', 'open', 'parents', 'people', 'person', 'pharisees', 'po',
            'post', 'potent', 'pretty', 'prof', 'purpose', 'pursue', 'rather',
            'really', 'recognise', 'referred', 'relationship', 'reliance',
            'results', 'right', 'rutgers', 'satan', 'secular', 'seem', 'sense',
            'share', 'she', 'since', 'sinless', 'someplace', 'soon', 'spend',
            'stay', 'steven', 'student', 'subjects', 'suited', 'sure',
            'symbol', 'technology', 'terrible', 'test', 'than', 'them', 'then',
            'things', 'think', 'through', 'tired', 'together', 'trait', 'true',
            'trusted', 'until', 'unto', 've', 'velasco', 'version', 'vice',
            'week', 'well', 'while', 'why', 'wont', 'wustl'], dtype='<U80')]

***

## __d) Sampling and log probs__

```python
# Make Multivariate Distribution

normal_distributions = tfd.MultivariateNormalDiag(loc=[[0.5, 1], [0.1, 0], [0, 0.2]],
                                 scale_diag=[[2, 3], [1, 3], [4, 4]])
normal_distributions
```

    <tfp.distributions.MultivariateNormalDiag 'MultivariateNormalDiag' batch_shape=[3] event_shape=[2] dtype=float32>

```python
# Sample

normal_distributions.sample(5)
```

    <tf.Tensor: shape=(5, 3, 2), dtype=float32, numpy=
    array([[[-0.61642456, -2.6508875 ],
            [-0.02484179, -0.58626443],
            [-2.3840659 , -2.2478259 ]],

           [[-1.3702699 ,  9.627965  ],
            [ 0.34466887, -2.335009  ],
            [-0.6538167 ,  5.7416673 ]],

           [[-2.086178  ,  4.7825446 ],
            [ 1.154147  , -4.501431  ],
            [-4.3120627 ,  0.32013714]],

           [[-2.5057547 ,  2.4638143 ],
            [-0.11568717, -5.0724864 ],
            [ 0.63433516, -0.52497053]],

           [[ 1.380878  ,  1.7464799 ],
            [ 0.11383934,  1.1534938 ],
            [-0.91430813,  2.0012252 ]]], dtype=float32)>

```python
# Multivariate Normal batched Distribution
# We are broadcasting batch shapes of `loc` and `scal_diag`
# against each other

loc = [[[0.3, 1.5, 1.], [0.2, 0.4, 2.8]],
        [[2., 2.3, 8], [1.4, 1, 1.3]]]
scale_diag = [0.4, 1., 0.7]
normal_distributions = tfd.MultivariateNormalDiag(loc=loc, scale_diag=scale_diag)
normal_distributions
```

    <tfp.distributions.MultivariateNormalDiag 'MultivariateNormalDiag' batch_shape=[2, 2] event_shape=[3] dtype=float32>

```python
# Use independent to move part of the batch shape

ind_normal_distributions = tfd.Independent(normal_distributions,
                                           reinterpreted_batch_ndims=1)
ind_normal_distributions
```

    <tfp.distributions.Independent 'IndependentMultivariateNormalDiag' batch_shape=[2] event_shape=[2, 3] dtype=float32>

```python
# Draw some samples

samples = ind_normal_distributions.sample(5)
samples.shape
```

    TensorShape([5, 2, 2, 3])

```python
# `[B, E]` shaped input

inp = tf.random.uniform((2, 2, 3))
ind_normal_distributions.log_prob(inp)
```

    <tf.Tensor: shape=(2,), dtype=float32, numpy=array([-11.756409, -66.098785], dtype=float32)>

```python
# `[E]` shaped input (broadcasting over batch size)

inp = tf.random.uniform((2, 3))
ind_normal_distributions.log_prob(inp)
```

    <tf.Tensor: shape=(2,), dtype=float32, numpy=array([-11.850761, -76.12896 ], dtype=float32)>

```python
#`[S, B, E]` shaped input (broadcasting over samples)

inp = tf.random.uniform((9, 2, 2, 3))
ind_normal_distributions.log_prob(inp)
```

    <tf.Tensor: shape=(9, 2), dtype=float32, numpy=
    array([[ -8.763502 , -74.403946 ],
           [ -8.963752 , -78.71576  ],
           [-10.053402 , -67.10482  ],
           [-10.113637 , -70.84384  ],
           [ -8.458472 , -77.99654  ],
           [-13.988488 , -74.964966 ],
           [ -7.66702  , -65.602325 ],
           [-10.51793  , -65.675446 ],
           [-11.9138565, -72.61048  ]], dtype=float32)>

```python
# `[S, b, e]` shaped input, where [b, e] is broadcastable over [B, E]

inp = tf.random.uniform((5, 1, 2, 1))
ind_normal_distributions.log_prob(inp)
```

    <tf.Tensor: shape=(5, 2), dtype=float32, numpy=
    array([[ -9.843089 , -61.68399  ],
           [-10.387953 , -61.109745 ],
           [ -9.2092285, -71.7383   ],
           [-12.003781 , -83.812454 ],
           [ -9.123208 , -68.52439  ]], dtype=float32)>

#### __Naive Bayes example__

Lets now use what we have learned and continue the Naive Bayes classifier we were building last tutorial.

#### Import the data

The dataset required for this coding tutorial can be downloaded from the following link:

[20_Newsgroup_Data.zip](https://github.com/mohd-faizy/Probabilistic-Deep-Learning-with-TensorFlow/raw/main/01_The%20TensorFlow_Probability_library/20_Newsgroup_Data.zip)

You should store this file in Drive for use in this Colab notebook.

```python
from sklearn.datasets import fetch_20newsgroups
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.naive_bayes import BernoulliNB
from sklearn.metrics import f1_score
```

```python
# Making a function get_data which:
#   1) Fetches the 20 newsgroup dataset
#   2) Performs a word count on the articles and binarizes the result
#   3) Returns the data as a numpy matrix with the labels

def get_data(categories):

    newsgroups_train_data = fetch_20newsgroups(data_home='/content/20_Newsgroup_Data',
                                               subset='train', categories=categories)
    newsgroups_test_data = fetch_20newsgroups(data_home='/content/20_Newsgroup_Data',
                                              subset='test', categories=categories)

    n_documents = len(newsgroups_test_data['data'])
    count_vectorizer = CountVectorizer(input='content', binary=True,max_df=0.25, min_df=1.01/n_documents)

    train_binary_bag_of_words = count_vectorizer.fit_transform(newsgroups_train_data['data'])
    test_binary_bag_of_words = count_vectorizer.transform(newsgroups_test_data['data'])

    return (train_binary_bag_of_words.todense(), newsgroups_train_data['target']),  (test_binary_bag_of_words.todense(), newsgroups_test_data['target'])
```

```python
# Defining a function to conduct Laplace smoothing. This adds a base level of probability for a given feature
# to occur in every class.

def laplace_smoothing(labels, binary_data, n_classes):
    # Compute the parameter estimates (adjusted fraction of documents in class that contain word)
    n_words = binary_data.shape[1]
    alpha = 1 # parameters for Laplace smoothing
    theta = np.zeros([n_classes, n_words]) # stores parameter values - prob. word given class
    for c_k in range(n_classes): # 0, 1, ..., 19
        class_mask = (labels == c_k)
        N = class_mask.sum() # number of articles in class
        theta[c_k, :] = (binary_data[class_mask, :].sum(axis=0) + alpha)/(N + alpha*2)

    return theta
```

```python
# Getting a subset of the 20 newsgroup dataset

categories = ['alt.atheism', 'talk.religion.misc', 'comp.graphics', 'sci.space']

(train_data, train_labels), (test_data, test_labels) = get_data(categories=categories)
smoothed_counts = laplace_smoothing(labels=train_labels, binary_data=train_data, n_classes=len(categories))
```

To now make our NB classifier we need to build three functions:
* Compute the class priors
* Build our class conditional distributions
* Put it all together and classify our data

```python
# Function which computes the prior probability of every class based on frequency of occurence in
# the dataset

def class_priors(n_classes, labels):
    counts = np.zeros(n_classes)
    for c_k in range(n_classes):
        counts[c_k] = np.sum(np.where(labels==c_k, 1, 0))
    priors = counts / np.sum(counts)
    print('The class priors are {}'.format(priors))
    return priors
```

```python
# Run the function

priors = class_priors(n_classes=len(categories), labels=train_labels)
```

    The class priors are [0.2359882  0.28711898 0.29154376 0.18534907]

```python
# Now we will do a function that given the feature occurence counts returns a Bernoulli distribution of
# batch_shape=number of classes and event_shape=number of features.

def make_distributions(probs):
    batch_of_bernoullis = tfd.Bernoulli(probs=probs)
    dist = tfd.Independent(batch_of_bernoullis,
                            reinterpreted_batch_ndims=1)
    return dist

tf_dist = make_distributions(smoothed_counts)
tf_dist
```

    <tfp.distributions.Independent 'IndependentBernoulli' batch_shape=[4] event_shape=[17495] dtype=int32>

```python
# The final function predict_sample which given the distribution, a test sample, and the class priors:
#   1) Computes the class conditional probabilities given the sample
#   2) Forms the joint likelihood
#   3) Normalises the joint likelihood and returns the log prob

def predict_sample(dist, sample, priors):
    cond_probs = dist.log_prob(sample)
    joint_likelihood = tf.add(np.log(priors), cond_probs)
    norm_factor = tf.math.reduce_logsumexp(joint_likelihood, axis=-1, keepdims=True)
    log_prob = joint_likelihood - norm_factor

    return log_prob
```

#### Computing log_probs

```python
# Predicting one example from our test data

log_probs = predict_sample(tf_dist, test_data[0], priors)
log_probs
```

    <tf.Tensor: shape=(4,), dtype=float32, numpy=
    array([-6.1736359e+01, -1.5258789e-05, -1.1619873e+01, -6.3327652e+01],
          dtype=float32)>

```python
# Loop over our test data and classify.

probabilities = []
for sample, label in zip(test_data, test_labels):
    probabilities.append(tf.exp(predict_sample(tf_dist, sample, priors)))

probabilities = np.asarray(probabilities)
predicted_classes = np.argmax(probabilities, axis =-1)
print('f1 ', f1_score(test_labels, predicted_classes, average='macro'))
```

    f1  0.7848499112849504

```python
# Make a Bernoulli Naive Bayes classifier using sklearn with the same level of alpha smoothing.

clf = BernoulliNB(alpha=1)
clf.fit(train_data, train_labels)
pred = clf.predict(test_data)
print('f1 from sklean ', f1_score(test_labels, pred, average='macro'))
```

    f1 from sklean  0.7848499112849504

***

## Trainable Distributions

```python
from sklearn.datasets import fetch_20newsgroups
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.naive_bayes import BernoulliNB
from sklearn.metrics import f1_score
```

```python
# Define an exponential distribution

exponential = tfd.Exponential(rate=0.3, name='exp')
```

```python
# Plot

plt.hist(exponential.sample(5000), bins=100, density=True)
plt.show()
```

```python
# Define an exponential distribution with a trainable rate parameter

exp_train = tfd.Exponential(rate=tf.Variable(1., name='rate'), name='exp_train')
exp_train.trainable_variables
```

    (<tf.Variable 'rate:0' shape=() dtype=float32, numpy=1.0>,)

```python
# Define the negative log likelihood

def nll(x_train, distribution):
    return -tf.reduce_mean(distribution.log_prob(x_train))

```

```python
# Define a function to compute the loss and gradients

@tf.function
def get_loss_and_grads(x_train, distribution):
    with tf.GradientTape() as tape:
        tape.watch(distribution.trainable_variables)
        loss = nll(x_train, distribution)
    grads = tape.gradient(loss, distribution.trainable_variables)
    return loss, grads
```

```python
# Optimize

def exponential_dist_optimisation(data, distribution):

    # Keep results for plotting
    train_loss_results = []
    train_rate_results = []

    optimizer = tf.keras.optimizers.SGD(learning_rate=0.05)

    num_steps = 10

    for i in range(num_steps):
        loss, grads = get_loss_and_grads(data, distribution)
        optimizer.apply_gradients(zip(grads, distribution.trainable_variables))

        rate_value = distribution.rate.value()
        train_loss_results.append(loss)
        train_rate_results.append(rate_value)
        print("Step {:03d}: Loss: {:.3f}: Rate: {:.3f}".format(i, loss, rate_value))

    return train_loss_results, train_rate_results
```

```python
# Get some data and train

sampled_data = exponential.sample(5000)
train_loss_results, train_rate_results = exponential_dist_optimisation(data=sampled_data, distribution=exp_train)
```

    Step 000: Loss: 3.392: Rate: 0.880
    Step 001: Loss: 3.114: Rate: 0.768
    Step 002: Loss: 2.868: Rate: 0.663
    Step 003: Loss: 2.660: Rate: 0.569
    Step 004: Loss: 2.494: Rate: 0.487
    Step 005: Loss: 2.372: Rate: 0.420
    Step 006: Loss: 2.292: Rate: 0.370
    Step 007: Loss: 2.249: Rate: 0.335
    Step 008: Loss: 2.230: Rate: 0.315
    Step 009: Loss: 2.224: Rate: 0.304

```python
# Predicted value for the rate parameter

pred_value = exp_train.rate.numpy()
exact_value = exponential.rate.numpy()

print("Exact rate: ", exact_value)
print("Pred rate:  ", pred_value)
```

    Exact rate:  0.3
    Pred rate:   0.30404457

```python
# Plot to see the convergence of the estimated and true parameters

tensor_exact_value = tf.constant(exact_value, shape=[len(train_rate_results)])

fig, axes = plt.subplots(2, sharex=True, figsize=(12, 8))
fig.suptitle('Convergence')

axes[0].set_ylabel("Loss", fontsize=14)
axes[0].plot(train_loss_results)

axes[1].set_ylabel("Rate", fontsize=14)
axes[1].set_xlabel("Epoch", fontsize=14)
axes[1].plot(train_rate_results, label='trainable rate variable')
axes[1].plot(tensor_exact_value, label='exact rate')
axes[1].legend()
plt.show()
```

```python
# Making a function get_data which:
#   1) Fetches the 20 newsgroup dataset
#   2) Performs a word count on the articles and binarizes the result
#   3) Returns the data as a numpy matrix with the labels

def get_data(categories):

    newsgroups_train_data = fetch_20newsgroups(data_home='/content/20_Newsgroup_Data',
                                               subset='train', categories=categories)
    newsgroups_test_data = fetch_20newsgroups(data_home='/content/20_Newsgroup_Data',
                                              subset='test', categories=categories)

    n_documents = len(newsgroups_test_data['data'])
    count_vectorizer = CountVectorizer(input='content', binary=True,max_df=0.25, min_df=1.01/n_documents)
    train_binary_bag_of_words = count_vectorizer.fit_transform(newsgroups_train_data['data'])
    test_binary_bag_of_words = count_vectorizer.transform(newsgroups_test_data['data'])

    return (train_binary_bag_of_words.todense(), newsgroups_train_data['target']),  (test_binary_bag_of_words.todense(), newsgroups_test_data['target'])

```

```python
# Defining a function to conduct laplace smoothing. This adds a base level of probability for a given feature
# to occur in every class.

def laplace_smoothing(labels, binary_data, n_classes):
    # Compute the parameter estimates (adjusted fraction of documents in class that contain word)
    n_words = binary_data.shape[1]
    alpha = 1 # parameters for Laplace smoothing
    theta = np.zeros([n_classes, n_words]) # stores parameter values - prob. word given class
    for c_k in range(n_classes): # 0, 1, ..., 19
        class_mask = (labels == c_k)
        N = class_mask.sum() # number of articles in class
        theta[c_k, :] = (binary_data[class_mask, :].sum(axis=0) + alpha)/(N + alpha*2)

    return theta
```

```python
# Now we will do a function that given the feature occurence counts returns a Bernoulli distribution of
# batch_shape=number of classes and event_shape=number of features.

def make_distributions(probs):
    batch_of_bernoullis = tfd.Bernoulli(probs=probs) # shape (n_classes, n_words)
    dist = tfd.Independent(batch_of_bernoullis, reinterpreted_batch_ndims=1)
    return dist
```

```python
# Function which computes the prior probability of every class based on frequency of occurence in
# the dataset

def class_priors(n_classes, labels):
    counts = np.zeros(n_classes)
    for c_k in range(n_classes):
        counts[c_k] = np.sum(np.where(labels==c_k, 1, 0))
    priors = counts / np.sum(counts)
    print('The class priors are {}'.format(priors))
    return priors
```

```python
# The final function predict_sample which given the distribution, a test sample, and the class priors:
#   1) Computes the class conditional probabilities given the sample
#   2) Forms the joint likelihood
#   3) Normalises the joint likelihood and returns the log prob

def predict_sample(dist, sample, priors):
    cond_probs = dist.log_prob(sample)
    joint_likelihood = tf.add(np.log(priors), cond_probs)
    norm_factor = tf.math.reduce_logsumexp(joint_likelihood, axis=-1, keepdims=True)
    log_prob = joint_likelihood - norm_factor

    return log_prob
```

```python
# Now we learn the distribution using gradient tape

def make_distribution_withGT(data, labels, nb_classes):

    class_data = []
    train_vars = []
    distributions = []
    for c in range(nb_classes):
        train_vars.append(tf.Variable(initial_value=np.random.uniform(low=0.01, high =0.1, size=data.shape[-1])))
        distributions.append(tfd.Bernoulli(probs=train_vars[c]))
        class_mask = (labels == c)
        class_data.append(data[class_mask, :])

    for c_num in range(0,nb_classes):
        optimizer = tf.keras.optimizers.Adam()
        print('\n%-------------------%')
        print('Class ', c_num)
        print('%-------------------%')

        for i in range(0,100):
            loss, grads = get_loss_and_grads(class_data[c_num], distributions[c_num])
            if(i%10==0):
                print('iter: {} loss: {}'.format(i, loss))
            optimizer.apply_gradients(zip(grads, distributions[c_num].trainable_variables))
            eta = 1e-3
            clipped_probs = tf.clip_by_value(distributions[c_num].trainable_variables,
                                             clip_value_min=eta, clip_value_max=1)
            train_vars[c_num] = tf.squeeze(clipped_probs)

    dist = tfd.Bernoulli(probs=train_vars)
    dist = tfd.Independent(dist,reinterpreted_batch_ndims=1)

    print(dist)

    return dist

```

```python
# Make the same Naive Bayes classifier we did last tutorial

categories = ['alt.atheism', 'talk.religion.misc', 'comp.graphics', 'sci.space']

(train_data, train_labels), (test_data, test_labels) = get_data(categories)

smoothed_counts = laplace_smoothing(labels=train_labels, binary_data=train_data, n_classes=len(categories))

priors = class_priors(n_classes=len(categories), labels=train_labels)
tf_dist = make_distributions(smoothed_counts)
```

    The class priors are [0.2359882  0.28711898 0.29154376 0.18534907]

```python
# Now train the distributions with gradient tape

GT_dist = make_distribution_withGT(data=train_data, labels=train_labels, nb_classes=4)
```

    %-------------------%
    Class  0
    %-------------------%
    iter: 0 loss: 0.07832579152423366
    iter: 10 loss: 0.06894347305528033
    iter: 20 loss: 0.060222996342706836
    iter: 30 loss: 0.05218120592196705
    iter: 40 loss: 0.044759031548345145
    iter: 50 loss: 0.0379018550873754
    iter: 60 loss: 0.03155126632645766
    iter: 70 loss: 0.02566911732741816
    iter: 80 loss: 0.020219924339709733
    iter: 90 loss: 0.0151572347232354

    %-------------------%
    Class  1
    %-------------------%
    iter: 0 loss: 0.07127955241503989
    iter: 10 loss: 0.061937990769672026
    iter: 20 loss: 0.05314094638202285
    iter: 30 loss: 0.0449779596289782
    iter: 40 loss: 0.03741778676096856
    iter: 50 loss: 0.030434644494345044
    iter: 60 loss: 0.023988602031351454
    iter: 70 loss: 0.018051947992478605
    iter: 80 loss: 0.012572028391125102
    iter: 90 loss: 0.007491437417768517

    %-------------------%
    Class  2
    %-------------------%
    iter: 0 loss: 0.0783904388365927
    iter: 10 loss: 0.06930189904430697
    iter: 20 loss: 0.06093254284122168
    iter: 30 loss: 0.05331069276547315
    iter: 40 loss: 0.046385938043994546
    iter: 50 loss: 0.04010591759581585
    iter: 60 loss: 0.034424281431691794
    iter: 70 loss: 0.029294317941405135
    iter: 80 loss: 0.02468416229071224
    iter: 90 loss: 0.020540241993693545

    %-------------------%
    Class  3
    %-------------------%
    iter: 0 loss: 0.07977267831453962
    iter: 10 loss: 0.07055890466877092
    iter: 20 loss: 0.06195969379331623
    iter: 30 loss: 0.053983481205388956
    iter: 40 loss: 0.046590926230770005
    iter: 50 loss: 0.03974021274615738
    iter: 60 loss: 0.03339246720568644
    iter: 70 loss: 0.027505496412316137
    iter: 80 loss: 0.0220337258491106
    iter: 90 loss: 0.016923285417091996
    tfp.distributions.Independent("IndependentBernoulli", batch_shape=[4], event_shape=[17495], dtype=int32)

```python
# Compare the two results

for dist in [GT_dist,tf_dist]:
    probabilities = []
    for sample, label in zip(test_data, test_labels):
        probabilities.append(predict_sample(dist, sample, priors))

    probabilities = np.asarray(probabilities)
    predicted_classes = np.argmax(probabilities, axis =-1)
    print('f1 ', f1_score(test_labels, predicted_classes, average='macro'))
```

    f1  0.8279055960457952
    f1  0.7848499112849504


# Linear regression
선형회귀 - 종속 변수 y와 한개 이상의 독립변수 X와의 선형관계를 모델링하는 방법론

```python
import sympy
import numpy

from matplotlib import pyplot
%matplotlib inline
sympy.init_printing()
```

```python
w = sympy.Symbol('w', real=True)

f = w**2 + 3*w - 5
f
```

```python
sympy.plotting.plot(f);
```

```python
fprime = f.diff(w)
fprime
```

```python
sympy.plotting.plot(fprime);
```

```python
sympy.solve(fprime, w)
```

# Gradient Descent

```python
fpnum = sympy.lambdify(w, fprime)
type(fpnum)
```

    function

```python
w = 10.0

for _ in range(1000):
  w = w - fpnum(w) *0.01

print(w)
```

    -1.4999999806458753

# 데이터셋 만들어보기

```python
x_data = numpy.linspace(-5, 5, 100)  #(시작점, 끝점, 점의 개수)
w_true = 2
b_true = 20

#y= wx+b 선분에 noise를 추가해서 출력(radom.normal 없으면 그냥 직선)
y_data = w_true*x_data + b_true + numpy.random.normal(size=len(x_data))
pyplot.scatter(x_data, y_data);
```

```python
x_data.shape
```

```python
y_data.shape
```

```python
w, b, x, y = sympy.symbols('w b x y')

cost_function = (w*x + b - y)**2
cost_function
```

```python
grad_b = sympy.lambdify([w,b,x,y], cost_function.diff(b), 'numpy')
grad_w = sympy.lambdify([w,b,x,y], cost_function.diff(w), 'numpy')
```

```python
w = 0
b = 0

for _ in range(1000):
  descent_b = numpy.sum(grad_b(w,b,x_data,y_data))/len(x_data)
  descent_w = numpy.sum(grad_w(w,b,x_data,y_data))/len(x_data)
  w = w - descent_w*0.01
  b = b - descent_b*0.01

print(w)
print(b)

```

    1.974233739486593
    19.949641406595283

```python
pyplot.scatter(x_data,y_data)
pyplot.plot(x_data, w*x_data + b, 'r');
```

```python
from IPython.display import YouTubeVideo
YouTubeVideo('gGOzHVUQCw0')
```

```python
from urllib.request import urlretrieve
URL = 'http://go.gwu.edu/engcomp1data5?accessType=DOWNLOAD'
urlretrieve(URL, 'land_global_temperature_anomaly-1880-2016.csv')
```

    ('land_global_temperature_anomaly-1880-2016.csv',
     <http.client.HTTPMessage at 0x7fd8346bffd0>)

```python
import numpy
```

```python
fname = '/content/land_global_temperature_anomaly-1880-2016.csv'

year, temp_anomaly = numpy.loadtxt(fname, delimiter=',', skiprows=5, unpack=True)
```

```python
from matplotlib import pyplot
%matplotlib inline
```

```python
pyplot.plot(year, temp_anomaly);
```

```python
pyplot.rc('font', family='serif', size='18')

#You can set the size of the figure by doing:
pyplot.figure(figsize=(10,5))

#Plotting
pyplot.plot(year, temp_anomaly, color='#2929a3', linestyle='-', linewidth=1)
pyplot.title('Land global temperature anomalies. \n')
pyplot.xlabel('Year')
pyplot.ylabel('Land temperature anomaly [°C]')
pyplot.grid();
```

```python
w = numpy.sum(temp_anomaly*(year - year.mean())) / numpy.sum(year*(year - year.mean()))
b = a_0 = temp_anomaly.mean() - w*year.mean()

print(w)
print(b)
```

    0.01037028394347266
    -20.148685384658464

```python
reg = b + w * year
```

```python
pyplot.figure(figsize=(10, 5))

pyplot.plot(year, temp_anomaly, color='#2929a3', linestyle='-', linewidth=1, alpha=0.5)
pyplot.plot(year, reg, 'k--', linewidth=2, label='Linear regression')
pyplot.xlabel('Year')
pyplot.ylabel('Land temperature anomaly [°C]')
pyplot.legend(loc='best', fontsize=15)
pyplot.grid();
```

```python

```


We consider the polar corrdinates in which (by mapping theorem),  raindrops are a point process with $\lambda r$ $drops/cm$
~~
$$
\begin{align}
P[\Lambda^1([0,r] \times [0, 1] \times R^2)=0] &=exp(-\int_{0}^r \int_{0}^{1} \int_{R^2}\lambda p(x,y)dxdydtdr)\\ &= exp(-\lambda t 2\pi \int_0^r \rho e^{-\rho}d\rho)\\
&= exp(-\lambda t 2\pi(1-e^{-r}(r+1)))\\
\end{align}
$$
~~


# HOMEWORK 4
## MATH 104C
## Amir Ostrowsky
## 5-22-2020

# 1.
Consider the standard second order finite difference approcimation to the second derivative
>$$D_0^2u_j=\frac{u_{j-1}-2u_j+u_{j+1}}{h^2}$$
Let $v_j^{(m)}=e^{imjh}$ for $j=1,\dots,M-1,\; h=\pi/M$

## 1.a.
Prove that $$D_0^2v_j^{(m)}=\frac{2}{h^2}(\cos(mh)-1)v_j^{(m)}.$$
In other words, $\lambda_m=\frac{2}{h^2}(\cos(mh)-1)$ is an eigenvalue of the tridiagonal matrix

$$A_{h} =
\frac{1}{h^2}\begin{bmatrix}
-2&1&0&&\dots&0\\
1&-2&1&0&\dots&0\\
&\ddots&\ddots&&\ddots&\\
0&\dots&0&1&-2&1\\
0&&\dots&0&1&-2\\
\end{bmatrix}
$$
associated with the eigenvector
$$\mathbf v^{(m)}=
\begin{bmatrix} v_1^{(m)}\\
v_2^{(m)}\\
\vdots\\
v_{M-1}^{(m)}\\
\end{bmatrix}$$

Proof.
$$\begin{align}
D_0^2v_j^{(m)}&=\frac{1}{h^2}\left(e^{im(j-1)h}-2e^{imjh}+e^{im(j+1)h}\right)\\
&=\frac{1}{h^2}\left(e^{imjh}(e^{-imh}+e^{imh}-2)\right)\\
&=\frac{1}{h^2}\left(e^{imjh}(2\cos(mh)-2)\right)\\
&=\frac{2}{h^2}(\cos(mh)-1)e^{imjh} \;\textrm{ with }\;v_j^{(m)}=e^{imjh}\\
&=\frac{2}{h^2}(\cos(mh)-1)v_j^{(m)}
\end{align}$$

with eigenvector $\mathbf v^{(m)}$ and tridiagonal matrix $A_h$ described above. $\quad\quad\blacksquare$

## 1.b.
Prove that the farthest eigenvalue fro the origin approaches $-4/h^2$ as $h \rightarrow 0$ and use this to explain why the ODE system
$$\frac{du_j(t)}{dt}=D_0^2u_j(t),\quad j=1,2,\dots, M-1$$
is stiff.

Proof.

The distance of the eigenvalue from the origin is $|\lambda_m - 0|$

then the maximum distance from the origin is $$\max_{m \in \mathbb R}\left|\frac{2}{h^2}(\cos(mh)-1)-0\right|=\left|\frac{2}{h^2}(\cos(\pi)-1)\right|=\left|\frac{2}{h^2}(-1-1)\right|=\frac{4}{h^2}$$

So, the farthest eigenvalue from the origin is $$\max_{m \in \mathbb R}|\lambda_m| = \left|\min_{m \in \mathbb R}\lambda_m\right|, \textrm{then } \min_{m \in \mathbb R}\lambda_m = -\frac{4}{h^2}.$$

and the minimum eigenvalue is $$\min_{m \in \mathbb R}\left|\frac{2}{h^2}(\cos(mh)-1)\right|=\left|\frac{2}{h^2}(\cos(2\pi)-1)\right|=\left|\frac{2}{h^2}(1-1)\right|=|0|=0$$

So the "stiffness ratio" is $$\frac{\max|\lambda_m|}{\min|\lambda_m|} = \lim_{h \rightarrow 0}\frac{\frac{4}{h^2}}{ {}\left|\frac{2}{h^2}(\cos(mh)-1)\right|} = \lim_{h\rightarrow 0 }\frac{2}{\cos(mh)-1} = +\infty$$

Since the "stiffness ratio" is not bounded from above then the ODE system is stiff.

#2.
**ADI.** COnsider the Heat equation in the unit square $[0,1]\times[0,1]$
$$u_t=u_{xx}+u_{yy}$$
with initial condition $$u(0,x,y)=\sin(\pi x)\sin(\pi y)$$
and the homogenous Dirichlet boundary condition $u=0$.

##2.a.
Implement the ADI method to find an approximation to this Initial boundary value problem. Determine $h$ and $k$ so that the numerical approximation is well resolved in both space and time.

```
'''
TRIDIAGONAL SQUARE MATIX SOLVER BY FACTORIZATION INTO
UPPER TRIANGULAR TIMES LOWER TRIANGULAR MATRICES LUfactorization
INPUT: tridiagonal matrix coefficients' arrays 'a', 'b', 'c', 'd'
'a' is the vector of diagonal entries of the matrix A,
'b' is the vector of diagonal entries above 'a',
'c' is the vector of diagonal entries below 'a',
'd' is the augmented vector of matrix A
OUTPUT: the vector Z of dimension (n-1 x 1) which are the second
derivatives of the domain nodes z_j for j = 0,...,n
from the equation Az = d

DERIVATIVE OF U WITH RESPECT TO 't': f_U()
INPUT:
'UjMinus1' is the array of entries above the diagonal,
'Uj' is the array of diagonal entries,
'UjPlus1' is the array of entries below the diagonal,
'alpha' is sigma constant times rate of change of time over rate
of change of x
OUTPUT: values of the Forward Euler ODE per x value

INITIAL VALUE CONDITION FUNCTION u_0():
INPUT:
'x' is value from the domain of 'X(x)'
OUTPUT: initial value of ODE at initial time step 't'= 0

VARIABLES, ARRAYS AND CONSTANTS
'ibc' is the initial boundry conditions,
'sigma' is the proportionality constant,
'maxX' is the maximal value of the domain of X,
'N' is the number of time values,
'M' is the number of domain values from X,
'T' is the is the maximal time value,
'h' is the rate of change of X,
'k' is the rate of change of time,
'alpha' is defined above,
'U' is matrix of approximation values of input variables 'x'
from the domain per time value 't'.
'''

import numpy as np
import math
import matplotlib
import matplotlib.pyplot as plt
import mpl_toolkits
from matplotlib import rc
from IPython.display import display, Math, Latex

# Say, "the default sans-serif font is Palatino Linotype"
matplotlib.rcParams['font.serif'] = "Palatino Linotype"
# Then, "ALWAYS use serif fonts"
matplotlib.rcParams['font.family'] = "DejaVu Sans"
# set the default Latex font for Latex text
plt.rcParams["mathtext.fontset"] = "cm"
# set default font size
matplotlib.rcParams.update({'font.size': 12})
```

```
def tridiagonalSolver(a,b,c,d):
    # factorization coefficients
    m = np.array([]); l = np.array([])
    m = np.append(m, a[0])
    n = int(a.size)
    for j in range(n-1):
        l = np.append(l, c[j] / m[j] )
        m = np.append(m, a[j+1]-l[j]*b[j])

    # Forward substitution on Ly = d
    y = np.array([])
    y = np.append(y, d[0])
    for j in range(1,n):
        y = np.append(y, d[j]-l[j-1]*y[j-1])

    # Backward substitution to solve Ux = y
    z = np.zeros(n)
    z[n-1] = y[n-1] / (m[n-1]+0.0)

    for j in range(n-2, -1, -1):
        z[j] = (y[j]-b[j]*z[j+1])/m[j]

    return z
```

```
sigma=0.003;maxX=1;
N=60; M=30; T=10.0;

# central difference approximation of 2nd derivative of
# U^n or U* (RHS) with respect to t
def D2uRHS(UjMinus1,Uj,UjPlus1,alpha):
    cent_diff = alpha*UjPlus1 +(1-2*alpha)*Uj + alpha*UjMinus1
    return(cent_diff)

def u_0(x,y):
    return (np.sin(np.pi*x)*np.sin(np.pi*y))

def ADI(u_0,sigma,D2uRHS,maxX,M,N,T):
  h=maxX/M # \Delta x and \Delta y
  k=T/N     # \Delta t
  r=k*sigma/(2*h**2) # coefficient input of tridiagonal matrices wrt x

  # initialize approximation of NxLxM matrix with Dirichlet boundary conditions
  U = np.full([N+1, M+1,M+1], 0.0)

  x,y = [j*h for j in range(M+1)],[j*h for j in range(M+1)]
  X,Y = np.meshgrid(x,y)

  # initialize conditions for first row, columns j = 0 to M-1,l = 0 to L-1
  U[0,:,:] = u_0(X,Y)

  # approximation at time n
  uStar = np.zeros((M+1,M+1))
  a = np.empty(M-1); a.fill(1+2*r)
  b = np.full([M-2],-r)

  for n in range(N):
    for i in range(1,M):
      # solving for u* -------------------------------------------------
      u_nl = [ D2uRHS(U[n,j-1,i],U[n,j,i],U[n,j+1,i],r) for j in range(1,M)]
      uStar[1:-1,i] = tridiagonalSolver(a,b,b,u_nl)

    for i in range(1,M):
      # solving for u^{n+1} --------------------------------------------
      d = [ D2uRHS(uStar[i,l-1],uStar[i,l],uStar[i,l+1],r) for l in range(1,M)]
      U[n+1,i,1:-1] = tridiagonalSolver(a,b,b,d)

  return X,Y,U
```

By lecture 12 **ADI** method is $\mathcal O(\Delta t)^2+\mathcal O(h^2)$ and is unconditionally stable. Hence it does not depend on $h$ and $k$ to be well resolved in time and space.

## 2.b.
Verify the accuracy of a scheme numerically,i.e., do a resolution study to estimate the order of convergence of the method.

```
h = maxX/20
X20,Y20,U20 = ADI(u_0,sigma,D2uRHS,maxX,20,N,T)
X40,Y40,U40 = ADI(u_0,sigma,D2uRHS,maxX,40,N,T)
X80,Y80,U80 = ADI(u_0,sigma,D2uRHS,maxX,80,N,T)

#plotADI(X50,Y50,U50,T,N, numPlots, elevation, rotation)
u20,u40,u80 = U20[30,10,10],U40[30,20,20],U80[30,40,40]
log2q = np.log2((u40-u20)/(u80-u40))

Latex("$\\mathcal O(2)= %f \\approx \\log(2^2)$" %np.round(log2q,2))
Latex("then ADI is of order $\\mathcal O(\\Delta x)^2$")

```

then ADI is of order $\mathcal O(\Delta x)^2$

## 2.c
Plot the numerical approximation at three times that illustrate well its time evolution.

```
# graphing
def plotADI(X,Y,U,T,N,h, numPlots, elevation, rotation):
  skipEvery = round(N/numPlots); k=T/N

  for i in range(0,N+1,30):
    fig = plt.figure()
    ax = fig.add_subplot (111, projection = '3d')
    ax.plot_surface(X, Y, U[i,:,:],
                    rstride=1, cstride=1,cmap='viridis',edgecolor='none')
    ax.set_xlabel("$y$")
    ax.set_ylabel("$x$")
    ax.set_zlabel('$U(x,y)$')
    ax.auto_scale_xyz([0, 1], [0, 1], [0, 1])
    title="$\\Delta x= %f, T=%i, \\; \\Delta t = %f,\\; t = %i$"
    ax.set_title(title %(h,T,k,k*i))
    ax.view_init(elevation,rotation)
    plt.tight_layout()
    plt.show()
```

```
numPlots=3;h=maxX/M # \Delta x and \Delta y
elevation, rotation = 60,60
X,Y,U = ADI(u_0,sigma,D2uRHS,maxX,M,N,T)
plotADI(X,Y,U,T,N, h, numPlots, elevation, rotation)

```


```python
import numpy as np
from scipy import io
from scipy import sparse
from scipy.sparse import csgraph
from scipy import fftpack
from scipy import signal
from scipy import linalg
from matplotlib import pyplot as plt
from matplotlib import collections
import mpl_toolkits.mplot3d.art3d as art3d
import seaborn as sns
import igraph

import graph3d

%matplotlib inline
```

```python
sns.set_palette('husl')
```

# Signal smoothness with respect to graph structure

```python
# Define a ring graph
n = 100
Aring = np.roll(np.eye(n), -1, axis=1) + np.roll(np.eye(n), 1, axis=1)

# Define a signal over the ring graph
f = np.sin(np.linspace(-8*np.pi, 8*np.pi, n))
```

```python
# Generate coords for plotting
x = np.cos(np.linspace(-2*np.pi, 2*np.pi, n))
y = np.sin(np.linspace(-2*np.pi, 2*np.pi, n))
xy = np.column_stack([x, y])
startcoords = xy[np.where(Aring)[0]]
endcoords = xy[np.where(Aring)[1]]
Aring_lines = [[tuple(x), tuple(y)] for x, y in zip(startcoords, endcoords)]
```

```python
Acrossed = Aring.copy()

# Randomly connect 40 vertices of the ring
s = np.random.randint(0, len(Aring), 40)
e = np.random.randint(0, len(Aring), 40)
for i, j in zip(s, e):
    Acrossed[i, j] = 1
    Acrossed[j, i] = 1

# Regenerate coords for plotting
startcoords = xy[np.where(Acrossed)[0]]
endcoords = xy[np.where(Acrossed)[1]]
Acrossed_lines = [[tuple(x), tuple(y)] for x, y in zip(startcoords, endcoords)]
```

```python
# Compute laplacian eigendecompositions
Lring = csgraph.laplacian(Aring)
lring, uring = linalg.eigh(Lring)
Lcrossed = csgraph.laplacian(Acrossed)
lcrossed, ucrossed = linalg.eigh(Lcrossed)
```

```python
# Compute graph Fourier transform of signals
fhat_ring = np.conj(uring.T) @ f
fhat_crossed = np.conj(ucrossed.T) @ f
```

```python
# Plot frequency spectrum for ring and crossed ring
fig = plt.figure(figsize=(10,6))

lc = collections.LineCollection(Aring_lines, linewidths=1, colors='k')
ax0 = fig.add_subplot(2, 2, 1, projection='3d')
graph3d.Graph3d(xy[:,0], xy[:,1], f, zmin=-4, zmax=4, fig=fig, ax=ax0, lc=lc)
ax0.set_title(r'$A_{ring}$')

lc = collections.LineCollection(Acrossed_lines, linewidths=1, colors='k')
ax1 = fig.add_subplot(2, 2, 3, projection='3d')
graph3d.Graph3d(xy[:,0], xy[:,1], f, zmin=-4, zmax=4, fig=fig, ax=ax1, lc=lc)
ax1.set_title(r'$A_{crossed}$')

ax2 = fig.add_subplot(2, 2, 2)
ax2.stem(lring, fhat_ring)
ax2.set_ylabel(r'$\hat{f}(\lambda)$')

ax4 = fig.add_subplot(2, 2, 4)
ax4.stem(lcrossed, fhat_crossed)
ax4.set_ylabel(r'$\hat{f}(\lambda)$')
ax4.set_xlabel(r'$\lambda$')

plt.tight_layout()
plt.savefig('../img/signal_smoothness_0.pdf', bbox_inches='tight')
```

# Local metrics of smoothness

#### The edge derivative of $f$ with respect to edge $e=(i,j)$

\begin{equation}
      \frac{\partial f}{\partial e} \bigg|_i = \sqrt{W_{i,j}} [f(j) - f(i)]
\end{equation}

  The local variation can be measured by the square root of the sum of the squared differences
  between signal values at adjacent vertices.

#### The graph gradient of $f$ at vertex $i$

\begin{equation}
    || \nabla_i f || = \bigg[ \bigg\{ \frac{\partial f}{\partial e} \bigg|_i \bigg\} \bigg|_{e \in \epsilon \ s.t. \ e=(i,j) \text{ for some } j \in V} \bigg]
\end{equation}

#### The local variation at vertex $i$
\begin{equation}
      || \nabla_i f ||_2 = \bigg[ \sum_{\text{e connected to i}} \bigg( \frac{\partial f}{\partial e} \bigg|_i \bigg)^2 \bigg]^{1/2} = \bigg[ \sum_{j \in N_i} W_{i,j} [f(j) - f(i)]^2 \bigg]^{1/2}
\end{equation}

```python
def graph_gradient(A, f, i):
    # j : unique neighbors of i (possibly asymmetrical)
    j = np.unique(np.concatenate([np.where(A[i, :])[0],
                                  np.where(A[:, i])[0]]))
    return A[i, j]*(f[i] - f[j])
```

```python
graph_gradient(Aring, f, 50)
```

    array([ 0.50229597, -0.43893102])

```python
def local_variation(A, f, i):
    local_var = np.sqrt(np.sum(graph_gradient(A, f, i)**2))
    return local_var
```

```python
local_variation(Aring, f, 50)
```

    0.66705448799508404

# Global metrics of smoothness

#### Discrete p-Dirichlet form of $f$
\begin{equation}
   S_p(f) = \frac{1}{p} \sum_{i \in V} \bigg[ \sum_{j \in N_i} W_{i,j} [f(j) - f(i)]^2 \bigg]^{\frac{p}{2}}
\end{equation}

For $p=1$, $S_1$ is simply the sum of local variations across all vertices.

For $p=2$, $S_2$ is a quadratic function of the Laplacian:

#### Graph Laplacian Quadratic Form
\begin{equation}
  \begin{split}
    S_2(f) = \frac{1}{2} \sum_{i \in V} \bigg[ \sum_{j \in N_i} W_{i,j}
        [f(j) - f(i)]^2 \bigg]^{\frac{1}{2}} = \sum_{(i, j) \in \epsilon}
        W_{i,j} [f(j) - f(i)]^2 \\
        = f^T L f
  \end{split}
\end{equation}

$S_2$ is small when $f$ has similar values at strongly-connected vertices.

```python
def dirichlet_p(A, f, p=2):
    i, j = np.where(A)
    return (1/p)*np.sum((A[i, j]*(f[i] - f[j])**2)**(p/2))
```

```python
def quad_laplace(L, f):
    return f @ L @ f
```

```python
# Is the dirichlet p form equivalent to the quadratic form?
dirichlet_ring = dirichlet_p(Aring, f, p=2)
quadratic_ring = quad_laplace(Lring, f)
np.isclose(dirichlet_ring, quadratic_ring)
```

    True

```python
dirichlet_crossed = dirichlet_p(Acrossed, f, p=2)
quadratic_crossed = quad_laplace(Lcrossed, f)
np.isclose(dirichlet_crossed, quadratic_crossed)
```

    True

```python
print('Graph laplacian quadratic form:')
print('\n')
print(r'Ring:         ', f @ Lring @ f)
print(r'Crossed ring: ', f @ Lcrossed @ f)
```

    Graph laplacian quadratic form:

    Ring:          12.4889116701
    Crossed ring:  56.6338735435


# Physics 420/580 Midterm Exam
## October 19, 2017 1pm-2pm

Do the following problems.  Use the Jupyter notebook, inserting your code and any textual answers/explanations in cells between the questions. (Feel free to add additional cells!)  Marks will be given based on how clearly you demonstrate your understanding.

There are no restrictions on downloading from the internet, eclass,  or the use of books, notes, or any other widely available computing resources.  However, **you are not allowed** to communicate with each other or collaborate in any way and uploading to the internet or sending or receiving direct communications is not appropriate.

When you are finished, upload the jupyter notebook to eclass. Eclass times out after 2:05 so make sure that you upload things before then.  Also be careful to save the notebook periodically and that you upload your final exam file.

```python
import numpy as np
import matplotlib.pyplot as plt
import matplotlib as mpl
import scipy
from scipy.integrate import odeint
from scipy.integrate import quad
from scipy.optimize import minimize
from scipy.optimize import fsolve
from scipy.optimize import least_squares
from scipy.interpolate import interp1d
from scipy.interpolate import CubicSpline

```

```python
mpl.rc('figure',dpi=250)
mpl.rc('text',usetex=True)
```

```python
def add_labels(xlabel, ylabel, title):
    plt.xlabel(xlabel)
    plt.ylabel(ylabel)
    plt.title(title)
    plt.legend()
```

## Graphics

Plot the two curves:
$$\begin{align} y&=4x^3-3x-2\\
x&=\sin(\frac{y^4}{4}-2y^2+2)
\end{align}
$$

How many intersections are there?  Read the x,y value corresponding to the intersections from the plots.

```python
fsolve(lambda x: x**3 - 1, 9)
```

    array([1.])

```python
"""
Plan:

-Initialize an array of xdata, ydata.
-Create two functions and then plot the two functions.
-Find the intersections by inspection, or if time allows, using fsolve/brute force.
- Read off the x, y values.
"""
```

```python
y1 = lambda x: 4*x**3 - 3*x - 2

xdata = np.linspace(-2, 2, 1000)
plt.plot(xdata, y1(xdata))

x1 = lambda y: np.sin(y**4/4 - 2*y**2 + 2)

ydata = np.linspace(-10, 10,1000)
plt.plot(x1(ydata), ydata)

curve1_x = xdata
curve1_y = y1(xdata)

curve2_x = x1(ydata)
curve2_y = ydata

tol = 1e-4
plt.ylim(-5, 2.5)
plt.xlim(-1.5, 1.5)
```

```python
from ipywidgets import interact, interact_manual

def plot(a):
    x = np.linspace(0, 5, 50)
    plt.plot(x, np.exp(a*x))
    plt.ylim(0, 5)

interact_manual(plot, a=5)
```

# Reconstruction

A instantaneous flash of light occurs in a large tank of water at time $t_0$ and at position $\vec{x_0}$

The group velocity of light in water is about $2.2 \times 10^8$ m/s.  Four sensors detect the light flash, and report a measurement of the time at which light strikes the sensor.  The locations of the sensors and the time which each sensor was hit is recorded in the table below:

|Sensor #|	x|	y|	z|		Time|
|--------|------------|------------|------------|------------|
|	|[m]|	[m]|	[m]|		[s]|
|1	|0	|0|	10	|	6.3859E-08|
|2	|8.66025404|	0|	-5|		1.1032E-07|
|3	|-4.33012702|	7.5|	-5|		7.9394E-08|
|4	|-4.33012702|	-7.5|	-5|		1.0759E-07|

Calculate the initial time $t_0$ and the location of the flash.

Here is a sketch of the general geometry.  Note that the gray region is not different from the rest of the water- it is just to show the tetrahedral arrangement of the light sensors.  The flash, shown as the yellow star is in an arbitrary location which you want to find.

```python
"""
Plan:

- Pick a random point called position. Calculate the distance to the each sensors, call it d
- The optimal point minimizes d - v*t for each sensor.
- Calculate the residuals
- Run least_squares on the residuals
- Find the optimal point and print it.
"""
```

```python
"cost function solution"

sen1 = np.array([0, 0, 10, 6.3859E-08])
sen2 = np.array([8.66025404,	0,	-5, 1.1032E-07])
sen3 = np.array([-4.33012702,	7.5,	-5,	7.9394E-08])
sen4 = np.array([-4.33012702,	-7.5,	-5,	1.0759E-07])

v = 2.2*10**8
def cost(pos):

    error = 0
    for sen in [sen1, sen2, sen3, sen4]:
        error += (np.linalg.norm(pos - sen[0:3]) - v*sen[3])**2

    return error

minimize(cost, np.random.randn(3))
```

          fun: 2.889558609330956
     hess_inv: array([[ 2.7050527 ,  1.63590179,  0.27960445],
           [ 1.63590179,  1.60674196, -0.18068716],
           [ 0.27960445, -0.18068716,  0.51680401]])
          jac: array([ 1.04308128e-06,  8.94069672e-08, -1.07288361e-06])
      message: 'Optimization terminated successfully.'
         nfev: 85
          nit: 12
         njev: 17
       status: 0
      success: True
            x: array([-7.8833527 , 10.57964442, 10.80804805])

```python
from scipy.optimize import least_squares
"residuals solution"

sen1 = np.array([0, 0, 10, 6.3859E-08])
sen2 = np.array([8.66025404,	0,	-5, 1.1032E-07])
sen3 = np.array([-4.33012702,	7.5,	-5,	7.9394E-08])
sen4 = np.array([-4.33012702,	-7.5,	-5,	1.0759E-07])

def cost(pos):
    error = np.zeros(4)
    for i, sen in enumerate([sen1, sen2, sen3, sen4]):
        error[i] = (np.linalg.norm(pos - sen[0:3]) - v*sen[3])

    return error

least_squares(cost, np.random.randn(3))
```

     active_mask: array([0., 0., 0.])
            cost: 1.4447793059407599
             fun: array([-0.83046438,  0.93904664, -0.97413042,  0.6075762 ])
            grad: array([-1.42800140e-05, -1.02340696e-05, -8.34587431e-07])
             jac: array([[-0.59639614,  0.80035917,  0.06112968],
           [-0.65625106,  0.41966651,  0.62706827],
           [-0.21545151,  0.1867243 ,  0.95849604],
           [-0.14636443,  0.74470816,  0.651143  ]])
         message: '`ftol` termination condition is satisfied.'
            nfev: 15
            njev: 15
      optimality: 1.4280013981682327e-05
          status: 2
         success: True
               x: array([-7.88347184, 10.57956012, 10.80804357])

```python
'''
Alternative solution

'''

class Sensor():
    def __init__(self, position, time):
        self.position = position
        self.time = time
        self.velocity = 2.2*10**8

    def draw_radius(self):
        r = self.velocity*self.time
        possible_vals = []
        for phi in np.linspace(0, 2*np.pi):
            for theta in np.linspace(0, np.pi):
                x = np.array([np.sin(theta)*np.cos(phi), np.sin(theta)*np.sin(phi), np.cos(theta)])
                possible_vals.append(self.position + r*x)
        return np.array(possible_vals)

```

```python
sen1 = Sensor(np.array([0, 0, 10]), 6.3859E-08)
sen2 = Sensor(np.array([8.66025404,	0,	-5]), 1.1032E-07)
sen3 = Sensor(np.array([4.33012702,	7.5,	-5]),	7.9394E-08)
sen4 = Sensor(np.array([-4.33012702,	-7.5,	-5]),	1.0759E-07)

circ = []
for i in [sen1, sen2, sen3, sen4]:
    circ.append(i.draw_radius())
```

```python
tol = 18
mask = np.abs(circ[0]-circ[1]-circ[2]-circ[3]) < tol

for i, j in enumerate(mask):
    if j.all() == True:
        print(i, j)
        print(circ[0][i])
```

    1010 [ True  True  True]
    [-7.04232267  4.58404406 21.25904395]
    1460 [ True  True  True]
    [-7.04232267 -4.58404406 21.25904395]

## Trajectories

An alpha-particle is a helium nucleus, with mass 6.644×10−27 kg or 4.002 u, and charge equal to twice the charge of an electron (but positive).  Calculate the trajectory of a 5MeV=(5*1.609 e-13 J) alpha particle as it moves by a gold nucleus (mass 196.966 u), with charge 79e, as a function of  the impact parameter b, below.  Assume both the alpha and the gold are point particles, and ignore special relativity.  Plot the scattering angle $\theta$ and energy loss of the alpha as a function of $b$, for values of b between 1e-16 and 1e-9m.

The force between two charge particles is given by the Coulomb potential:
$$\vec{F}=\frac{1}{4\pi\epsilon_0}\frac{q_1q_2 (\vec{r_2}-\vec{r_1})}{|\vec{r_2}-\vec{r_1}|^3}$$ with $\epsilon_0=8.85\times10^{-12}\frac{\rm{C}^2}{\rm{N\cdot m^2}}$

```python
"""
Plan:

- Write NEwtons law as system of coupled ODE and integrate.
- F is couloumbs law, so the acceleration is F/m
-

"""
```

```python
eps = 8.85*10**-12
elementary_charge = 1.60*10**-19

q1 = 2*elementary_charge
q2 = 79*elementary_charge

u = 1.66*10**-27
m1 = 4*u
m2 = 196.966*u

def CouloubsLaw(r1, r2, q1, q2):
    distance = np.linalg.norm(r2-r1)
    direction = (r2-r1)/distance
    F = (1/(4*np.pi*eps))*(q2/distance**2)*direction
    return F
```

```python
kinetic = 5*1.609e-13
velocity = np.sqrt(2*kinetic/m1)
velocity
```

    15566607.758546297

```python
import numpy as np
from scipy.integrate import odeint

def main(y, t, m1 = m1, m2 = m2):
    '''
    Main function to be integrated with odeint.
    Input: y (array of elements 12), t (scalar)
    Output: dydt (array of elements 12)
    '''

    particle1_x = y[0:2]
    particle1_dx = y[2:4]

    particle2_x = np.array([0, velocity])

    F = CouloubsLaw(particle1_x, particle2_x, q1, q2)

    dxdt = np.zeros(4)

    dxdt[0:2] = particle1_dx
    dxdt[2:4] = F/m1

    #print(y[0:2])

    return dxdt

b = 1e-16
y0 = np.array([0, b, velocity, 0])
t = np.linspace(0, 3, num=500)
y = odeint(main, y0, t)

```

```python
y

plt.plot(y[:,0], y[:,1])
```

```python
energy = 1/2*m1*y[:, 2]**2 + 1/2*m1*y[:, 3]**2
plt.plot(t, energy)
```

```python
for b in np.logspace(-16, -9, num=10):
    y0 = np.array([0, b, velocity, 0])
    t = np.linspace(0, 3, num=500)
    y = odeint(main, y0, t)
    plt.plot(y[:,0], y[:,1], label=b)

add_labels('x', 'y', 'Scattering for various b')
```

```python
for b in np.logspace(-16, -9, num=10):
    y0 = np.array([0, b, velocity, 0])
    t = np.linspace(0, 3, num=500)
    y = odeint(main, y0, t)

    energy = 1/2*m1*y[:, 2]**2 + 1/2*m1*y[:, 3]**2
    plt.plot(t, energy, label = b)
    add_labels('x', 'y', 'Scattering for various b')
```

```python

```

```python
def f(x):
    return np.array([x**2, np.sin(x)])

xdata = np.linspace(0, 10, num=10)
ydata = f(xdata)

g = scipy.interpolate.interp1d(xdata, ydata, kind='cubic')
```

```python
x = np.linspace(0, 10)

plt.plot(x, g(x)[0])
plt.plot(x, g(x)[1])
```

```python

```

# Don't forget to upload your work to eclass!

```python

```