# Abstract

4Chan is famous for it's anonymous and ephemeral boards. But is it really ephemeral? There are multiple internet archiving services that have been keeping copies of 4chan posts that go many years in the past. We looked into it to asnwer the following questions: Q1.) *Are 4chan archiving services reliable and offer a holistic view of the epehemeral 4chan threads?* q2.) *If (Q1) is true, does this provides any insights into the pseudo anonymity of the site?*

```python
%%capture
# needed to support mongo+srv connection strings
!pip install dnspython
!pip install pymongo
!pip install pandas
!pip install numpy
!pip install matplotlib
!pip install scipy
!pip install gensim
!pip install langdetect
!pip install tqdm
!pip install nltk
!pip install seaborn
!pip install pyLDAvis
!pip install wordcloud

## Only run once per kernel environment
## Doesn't need to be re-run when kernel is restarted
#import nltk
#nltk.download('punkt')
#nltk.download('averaged_perceptron_tagger')
#nltk.download('wordnet')
#nltk.download('stopwords')

```

```python
%%capture

from pymongo import MongoClient
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np

# Fixed period analysis - change these to extend/restrict the period
# Date as a yyyymmddhhMMss string
date_from = '202105010000'
date_to =   '202111010000'

# connectiong to mongodb cluster using a read only user

## REMOTE REPLICA SET
client = MongoClient("mongodb+srv://<usr>:<pass>@rs.prifti.us/?ssl=false&authSource=chan")

## LOCAL SINGLE INSTANCE
## 192.168.1.110 -- localhost
## client = MongoClient("mongodb://<usr>:<pass>@localhost:27777/?ssl=false&serverSelectionTimeoutMS=5000&connectTimeoutMS=10000&authSource=chan&authMechanism=SCRAM-SHA-256")

# connection to chan database
db = client.chan

# the threads collection contains API data collections from the 4chan api and 4pleb api
threads = db['y-threads']

# the 'live-4chan' collection contains the OXPath data collection
livechan = db['y-live-4chan']

```

```python
# %%capture

# MongoDB aggregation query, joining data from thread and 'live-4chan' collection
# for each discovered 4chan thread - retreive a boolean representing if the API data was found
# Fixed period analysis - change these to extend/restrict the period
# Date as a yyyymmddhhMMss string
date_from = '202105010000'
date_to =   '202111010000'

q1_pipeline = [
    # normalise nulls in $data
    { "$addFields": { "data": { "$cond": [
        {"$and":[ {"$gt": ["$data", None] }, {"data": {"$type": "array"} } ] }, "$data", [{ }]
      ] } } }
    # only load required fields from 'live-4chan' collection
    ,{ "$project": { "_id": 0, "url": 1, "deleted": "$attributes.deleted", "archived": "$attributes.archived"
                    ,"first_scan": {"$arrayElemAt": ["$scans", 0 ] }
                    ,"last_scan": {"$arrayElemAt": ["$scans", -1 ] }
                    ,"oxp_op": { "$arrayElemAt": [ "$data", -1 ] }
                  }
    }

    # limit the data to fixed period
   ,{ "$match": {"$and":[ {"first_scan": { "$gt": date_from} }, {"last_scan": { "$lt": date_to }} ] } }
    # join with 'threads' collection on thread unique URL
   ,{ "$lookup": {
               "from": "y-threads",
               "localField": "url",
               "foreignField": "_id",
               "as": "api_thread"
       }
    }
   # unwind/expand the joined array
  ,{ "$unwind": "$api_thread" }
   # remove items where the API scrap job has not been performed
   # this should only affects threads that have not been archived/deleted yet
  ,{ "$match": { "api_thread.api_scrap": True } }
   #,{ "$project": { "url": 1 }}
   # include the op thread scaped via 4Chan API
  ,{ "$addFields": { "chan_op": { "$cond": [
        {"$and":[ {"$gt": ["$api_thread.chan.posts", None] }] }, "$api_thread.chan.posts", [{ }]
      ] } } }
   # add the boolean fields that check if chan api fields has data
  ,{ "$project": { "_id": 0, "url": 1, "deleted": 1, "archived": 1, "last_scan": 1
                    ,"oxp_op": "$oxp_op.payload"
                    ,"chan_op": {"$arrayElemAt": ["$chan_op", 0]}
                    ,"chan": {"$gt": [ "$api_thread.chan.posts", { } ] }
                    ,"pleb": "$api_thread.pleb"
        }
   }
   # keep the OP/thread from OXpath scrap
  ,{"$unwind": {"path": "$oxp_op", "preserveNullAndEmptyArrays": True } }
   # convert a document to array to better check relevant fields have values
  ,{ "$addFields": {
        "pleb_array": {
                "$ifNull": [ {"$objectToArray": "$pleb"}, [ ] ]
          }
     }
   }
   # bring array fields one level up
  ,{ "$unwind": { "path": "$pleb_array", "preserveNullAndEmptyArrays": True } }
   # check the pleb api has data
  ,{ "$project": { "_id": 0, "url": 1, "deleted": 1, "archived": 1, "last_scan": 1,
                    "oxp_op":"$oxp_op.thread.op", "oxp": { "$gt": ["$oxp_op", {} ] }
                  ,"chan": 1, "chan_op": 1
                  ,"pleb_op": "$pleb_array.v.op"
                  ,"pleb": { "$gt": [ "$pleb_array.v.op", { } ] } }
   }
  ,{ "$unwind": {"path": "$oxp_op", "preserveNullAndEmptyArrays": True } }
  ,{ "$unwind": {"path": "$oxp_op", "preserveNullAndEmptyArrays": True } }
 #,{ "$group": { _id: null, "count": { $sum: 1 } } }
  ,{ "$sort": {"last_scan": -1} }
]

# run the aggregation
match_result = pd.DataFrame(livechan.aggregate(q1_pipeline, allowDiskUse=True))

```

```python
match_result
```

      url
      deleted
      archived
      last_scan
      chan_op
      chan
      oxp
      pleb_op
      pleb
      oxp_op

      0
      https://boards.4chan.org/pol/thread/345511307
      False
      True
      20211031235904
      {}
      False
      False
      {'doc_id': '321648245', 'num': '345511307', 's...
      True
      NaN

      1
      https://boards.4chan.org/pol/thread/345544668
      False
      True
      20211031235904
      {}
      False
      False
      {'doc_id': '321681512', 'num': '345544668', 's...
      True
      NaN

      2
      https://boards.4chan.org/pol/thread/345545223
      False
      True
      20211031235904
      {}
      False
      False
      {'doc_id': '321682079', 'num': '345545223', 's...
      True
      NaN

      3
      https://boards.4chan.org/pol/thread/345545415
      False
      True
      20211031235904
      {}
      False
      False
      {'doc_id': '321682252', 'num': '345545415', 's...
      True
      NaN

      4
      https://boards.4chan.org/pol/thread/345537604
      False
      True
      20211031235805
      {}
      False
      False
      {'doc_id': '321674445', 'num': '345537604', 's...
      True
      NaN

      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...

      407931
      https://boards.4chan.org/pol/thread/319461189
      False
      False
      20210501002857
      {}
      False
      True
      {'doc_id': '295621738', 'num': '319461189', 's...
      True
      {'image': ['//i.4cdn.org/pol/1619828463836.jpg...

      407932
      https://boards.4chan.org/pol/thread/319459954
      False
      False
      20210501002656
      {'no': 319459954, 'closed': 1, 'now': '04/30/2...
      True
      True
      {'doc_id': '295620477', 'num': '319459954', 's...
      True
      {'image': ['//i.4cdn.org/pol/1619827838385.jpg...

      407933
      https://boards.4chan.org/pol/thread/319458666
      False
      False
      20210501002156
      {'no': 319458666, 'closed': 1, 'now': '04/30/2...
      True
      True
      {'doc_id': '295619151', 'num': '319458666', 's...
      True
      {'image': ['//i.4cdn.org/pol/1619827175027.png...

      407934
      https://boards.4chan.org/pol/thread/319458870
      False
      False
      20210501001911
      {'no': 319458870, 'closed': 1, 'now': '04/30/2...
      True
      True
      {'doc_id': '295619448', 'num': '319458870', 's...
      True
      {'image': ['//i.4cdn.org/pol/1619827276328.jpg...

      407935
      https://boards.4chan.org/pol/thread/319458755
      False
      False
      20210501001851
      {'no': 319458755, 'closed': 1, 'now': '04/30/2...
      True
      True
      {'doc_id': '295619230', 'num': '319458755', 's...
      True
      {'image': ['//i.4cdn.org/pol/1619827216254.png...

407936 rows × 10 columns

```python
# %%capture

## Length of the dataset where there is data from 4Chan API (chan), 4Pleb API (pleb) and OXPath Scrap (oxp)
full_match_len = len(match_result.query("chan & pleb & oxp" ))

print(f'There are **{len(match_result)}** threads crawled during the period in question. ')
print(f'Of these there are **{match_result.pleb.sum()}** threads that have been found in 4pleb archiving API. This is equivalent to **{"%.3f" % (match_result.pleb.sum()/len(match_result)*100)}%** match.')
print(f'Nr of matching threads we have collected data from all three sources (OXPATH, 4CHAN, 4PLEB): {full_match_len}, that is **{"%.3f" % (full_match_len/len(match_result)*100)}%** ')

# Select only items that have not been found on 4pleb
f1 = match_result[match_result.pleb==False]
# Of these, select only items that have not been scaped from 4chan
f2 = f1 # ^ paused - f2 used as f1 later - do not change
# Of these, select the items where we have a copy of the data from oxpath but not found on 4chan and 4pleb
f3 = f2[f2.oxp == True]

# Show the data  from original post scraped using OXPath
oxp_op = pd.DataFrame(list(f3['oxp_op']))
post_date = oxp_op.explode('date_time')

post_date['date_time'] = pd.to_datetime(post_date['date_time'], format='%m/%d/%y(%a)%X')
post_date=post_date.sort_values(by='date_time')

dates = post_date.date_time

times = pd.DatetimeIndex(dates)
grp = dates.groupby([times.hour])

print('Plot showing the distribution of missed threads through out the 24 hour (UK Time)')

cnt = grp.agg(['count'])

my_df = pd.DataFrame([cnt.index, cnt.values])
my_df = my_df.transpose()
my_df.columns = ['time_of_day','nr_missed_threads']
my_df = my_df.explode('nr_missed_threads')
#my_df
#grouped.agg(['count'])
plt.hist(my_df.time_of_day, bins=24, weights=my_df.nr_missed_threads)
#np.mean(my_df.nr_missed_threads)

```

```python
# %%capture

import scipy.stats as st
import statistics as stat

def rolling(time_series):
  diff, _d_from, _d_to = [],[],[]
  for i in range(1, len(time_series)-1):
      temp_diff = (time_series[i] - time_series[i-1]).total_seconds() / 60
      diff.append(temp_diff)
      _d_from.append(time_series[i-1])
      _d_to.append(time_series[i])
  return pd.DataFrame({"Gap": diff, "From": _d_from, "To": _d_to})

sorted_freq = freq_dataset#.sort_values(by=["scan_num"])
count = sorted_freq['count']
scan = list(sorted_freq['scan'])
_to_date = scan[0]
_from_date = scan[len(scan) - 1]
_minutes = (_from_date - _to_date).total_seconds() / 60
_total_scans = count.sum()

print(f'Total scans {len(freq_dataset)}')
print(f'First Scan: {_from_date}, Last Scan: {_to_date}, Total Minutes {_minutes}')
print(f"Total Scans {_total_scans}, Avg scans per minute {'%.2f' % (_total_scans/_minutes )}")
print(f'Scan frequency mean {"%.2f" % stat.mean(count)}, geometric mean {"%.2f" % st.gmean(count)}, \
median {stat.median(count)}, std {"%.2f" % stat.stdev(count)}')
rolling(scan).sort_values(by=["Gap"], ascending=False).head(10)

perc_match = (match_result.pleb.sum()/len(match_result))*100
perc_not_match = (1-(match_result.pleb.sum()/len(match_result)))*100
not_pleb_match = match_result.query("pleb==False")
not_pleb_yes_chan = not_pleb_match.query("chan")

result_rolling = rolling(scan).sort_values(by=["Gap"], ascending=False)#.head(10)
result_rolling
```

    Total scans 2516
    First Scan: 2022-04-17 17:57:00, Last Scan: 2022-04-16 00:01:00, Total Minutes 2516.0
    Total Scans 10942, Avg scans per minute 4.35
    Scan frequency mean 4.35, geometric mean 4.20, median 4.0, std 1.04

      Gap
      From
      To

      698
      2.0
      2022-04-16 11:39:00
      2022-04-16 11:41:00

      1669
      1.0
      2022-04-17 03:51:00
      2022-04-17 03:52:00

      1671
      1.0
      2022-04-17 03:53:00
      2022-04-17 03:54:00

      1672
      1.0
      2022-04-17 03:54:00
      2022-04-17 03:55:00

      1673
      1.0
      2022-04-17 03:55:00
      2022-04-17 03:56:00

      ...
      ...
      ...
      ...

      839
      1.0
      2022-04-16 14:01:00
      2022-04-16 14:02:00

      840
      1.0
      2022-04-16 14:02:00
      2022-04-16 14:03:00

      841
      1.0
      2022-04-16 14:03:00
      2022-04-16 14:04:00

      842
      1.0
      2022-04-16 14:04:00
      2022-04-16 14:05:00

      2513
      1.0
      2022-04-17 17:55:00
      2022-04-17 17:56:00

2514 rows × 3 columns

```python
# %%capture

not_pleb_not_chan_oxp = not_pleb_match.query("chan==False & oxp==True")
not_pleb_yes_chan_or_oxp = not_pleb_match.query("chan | (chan==False & oxp==True)")
print(f'Nr items that are not in 4pleb but are in 4chan {(len(not_pleb_yes_chan)/len(not_pleb_match))*100}')

def get_num(url: str) -> int:
    return int(url.split('/')[-1])

def get_num_str(url: str) -> str:
    return url.split('/')[-1]

pleb_qry = match_result.query("pleb & deleted==True")
chan_pleb_qry = pleb_qry.query("chan")
# Show the data  from original post
pleb_posts = pd.DataFrame(list(pleb_qry['pleb_op']))
chan_posts = pd.DataFrame(list(chan_pleb_qry['chan_op']))
print('Pleb Deleted {}'.format(len(pleb_posts.query("deleted=='1'"))))
print(f'Oxp Deleted {len(pleb_posts)}')
id_chan_posts = list(chan_posts.query('filedeleted != "NaN"')['no'])
match_result['num'] = match_result['url'].apply(get_num)
#match_result
num_del_chan = len(match_result[match_result['num'].isin(id_chan_posts)].query('deleted==True'))

id_pleb_posts = pleb_posts.query("deleted=='1'")['num']
match_result['num'] = match_result['url'].apply(get_num_str)
print(len(id_pleb_posts))
print(len(match_result[match_result['num'].isin(id_pleb_posts)].query('deleted==True')))

```

    Nr items that are not in 4pleb but are in 4chan 34.946236559139784
    Pleb Deleted 6600
    Oxp Deleted 11054
    6600
    6600

# Introduction

We have been collecting posts from the the /pol board of 4chan with high frequency (scaning the board multiple times per minute).

We have done so by simulating an end user that connects to the side with a conventional borwser (Firefox), using [OXPath](https://github.com/oxpath/oxpath)

We have been taking multiple snapshots of each thread until the thread was archived. When a thread is archived we then have been taking a snapshot of the thread from the 4chan api and one from 4pleb.org, an 4chan archiving service, believed to be the most complete archiving service on 4chan.

Two type of agents have been used to collect the data using OXpath.

- A discovery crawling agent that continuously crawls the /pol board looking for new threads and either a.) captures and queues the new thread URLs for scraping or b.) updates the *found* timestamp of a thread that had already been found by previous instances of the crawler.

- A extraction scraping agent that continuously scrapes threads queued by the crawling agent and stores the state of the thread together with the timestamp, unless the agent finds the thread to be *archived* or *deleted*. When the thread is found in the *archived* / *deleted* state, its attributes are updated and the thread is removed from the crawling queue.

# Data overview

Whilst there is a continous stream of data, to ensure reproducibility, this analisys is specifically to the dates between {{date_from}} and {{date_to}}.

The data collected by scanning the 4chan /pol board has the following caracteristics:

|Property   | Value   | Description   |
|---|---|---|
|Total Threads   |{{len(match_result)}}   | The number of total threads found   |
|Total 4Pleb Threads   |{{match_result.pleb.sum()}}  / {{"%.3f" % (match_result.pleb.sum()/len(match_result)*100)}}% | Number/Percentage of threads also found on 4Pleb API  |
|Three way Matched Threads   | {{full_match_len}} / {{"%.3f" % (full_match_len/len(match_result)*100) }}% | The number of threads where we have collected data from all three sources  |
|First-Last scan | {{ _from_date.strftime("%d/%m/%Y, %H:%M:%S") }} **-** {{ _to_date.strftime("%d/%m/%Y, %H:%M:%S") }}  | The first and the last scan in the dataset   |
|Total Minutes   | {{ _minutes }}   | The total minutes elapsed between the two dates   |
|Total Scans   | {{_total_scans}}   | The total number of scans seen on the dataset. That is the number of times we crawled the /pol board to find new threads   |
|Scans per minute| {{'%.2f' % (_total_scans/_minutes )}}  | The avg scans per minute based on the total minutes elapsed between the two dates |
|Frequency mean| {{"%.2f" % stat.mean(count)}}  | The avg scans per minute based on the minutes where there has been scans (there are gaps in the dataset) |
|Median | {{stat.median(count)}}   | The median value of scans per minute |
|Standard Deviation| {{"%.2f" % stat.stdev(count)}} | The std of observed scans per minute |

```python
# %%capture

deleted_threads = match_result.query('pleb').query("deleted==True")
deleted_threads = pd.DataFrame(list(deleted_threads['pleb_op']))
deleted_threads = deleted_threads.drop(columns=['doc_id', 'subnum', 'thread_num', 'op', 'capcode', 'email', 'name', 'exif'])
deleted_threads = deleted_threads.drop(columns=['trip','sticky', 'locked', 'poster_hash_processed', 'poster_country_name_processed', 'troll_country_code', 'troll_country_name', 'since4pass', 'unique_ips', 'extra_data', 'media', 'board'])
deleted_threads = deleted_threads.drop(columns=['trip_processed', 'email_processed', 'name_processed', 'formatted'])
#document_corp = ', '.join(list(deleted_threads.query('title != None')['title']))

#len(deleted_threads)
#len(deleted_threads.query('deleted=="1"'))
deleted_threads = deleted_threads.query('deleted=="0"')
```

```python
# %%capture

deleted_threads['articles'] = deleted_threads['title'] + ' ' + deleted_threads['comment']
document_corp = list(deleted_threads['articles'])
document_corp = [x for x in document_corp if x is not None]
document_corp = pd.DataFrame(document_corp)
#document_corp.shape
document_corp.columns = ['articles']
document_corp = document_corp.dropna().reset_index(drop=True)
#temp_title = list(deleted_threads.query('title != None')['title'])
#temp_title = [x for x in temp_title if x is not None]

#temp_comments = list(deleted_threads.query('comment != None')['comment'])
#temp_comments = [x for x in temp_comments if x is not None]
#document_corp = temp_title + temp_comments
#len(document_corp)

```

```python
# %%capture

from langdetect import detect
from tqdm import notebook
notebook.tqdm().pandas()
data = document_corp.copy()

def try_detect(elem):
    try:
        return detect(elem)
    except:
        return 'n/a'

data['lang'] = data.articles.progress_map(try_detect)
data['lang'].head(5)

```

    HBox(children=(HTML(value=''), FloatProgress(value=1.0, bar_style='info', layout=Layout(width='20px'), max=1.0…

    HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=2243.0), HTML(value='')))

    0    en
    1    en
    2    en
    3    en
    4    en
    Name: lang, dtype: object

```python
# %%capture

from nltk.tokenize import sent_tokenize

data.lang.value_counts()
data_en = data.loc[data.lang=='en']
data['sentences'] = data.articles.progress_map(sent_tokenize)
data['sentences'].head(1).tolist()[0][:3] # Print the first 3 sentences of the 1st article
```

    HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=2243.0), HTML(value='')))

    ['Australia health orders This thread is political as it is about current politically motivated restrictions in Australia.',
     "Aus bros I need some help here, I hear a lot about legal restrictions and confronting police about their legality (I'm in Vic but whatever state).",
     'Can you fellas help me and any other aussies with some solid things to say to cops and any advice in general?']

```python
# %%capture

from nltk.tokenize import word_tokenize
data['tokens_sentences'] = data['sentences'].progress_map(lambda sentences: [word_tokenize(sentence) for sentence in sentences])
print(data['tokens_sentences'].head(1).tolist()[0][:3])

```

    HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=2243.0), HTML(value='')))

    [['Australia', 'health', 'orders', 'This', 'thread', 'is', 'political', 'as', 'it', 'is', 'about', 'current', 'politically', 'motivated', 'restrictions', 'in', 'Australia', '.'], ['Aus', 'bros', 'I', 'need', 'some', 'help', 'here', ',', 'I', 'hear', 'a', 'lot', 'about', 'legal', 'restrictions', 'and', 'confronting', 'police', 'about', 'their', 'legality', '(', 'I', "'m", 'in', 'Vic', 'but', 'whatever', 'state', ')', '.'], ['Can', 'you', 'fellas', 'help', 'me', 'and', 'any', 'other', 'aussies', 'with', 'some', 'solid', 'things', 'to', 'say', 'to', 'cops', 'and', 'any', 'advice', 'in', 'general', '?']]

```python
# %%capture

from nltk import pos_tag
data['POS_tokens'] = data['tokens_sentences'].progress_map(lambda tokens_sentences: [pos_tag(tokens) for tokens in tokens_sentences])
print(data['POS_tokens'].head(1).tolist()[0][:3])
```

    HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=2243.0), HTML(value='')))

    [[('Australia', 'NNP'), ('health', 'NN'), ('orders', 'NNS'), ('This', 'DT'), ('thread', 'NN'), ('is', 'VBZ'), ('political', 'JJ'), ('as', 'IN'), ('it', 'PRP'), ('is', 'VBZ'), ('about', 'IN'), ('current', 'JJ'), ('politically', 'RB'), ('motivated', 'JJ'), ('restrictions', 'NNS'), ('in', 'IN'), ('Australia', 'NNP'), ('.', '.')], [('Aus', 'NNP'), ('bros', 'NN'), ('I', 'PRP'), ('need', 'VBP'), ('some', 'DT'), ('help', 'NN'), ('here', 'RB'), (',', ','), ('I', 'PRP'), ('hear', 'VBP'), ('a', 'DT'), ('lot', 'NN'), ('about', 'IN'), ('legal', 'JJ'), ('restrictions', 'NNS'), ('and', 'CC'), ('confronting', 'VBG'), ('police', 'NNS'), ('about', 'IN'), ('their', 'PRP$'), ('legality', 'NN'), ('(', '('), ('I', 'PRP'), ("'m", 'VBP'), ('in', 'IN'), ('Vic', 'NNP'), ('but', 'CC'), ('whatever', 'WDT'), ('state', 'NN'), (')', ')'), ('.', '.')], [('Can', 'MD'), ('you', 'PRP'), ('fellas', 'VB'), ('help', 'VB'), ('me', 'PRP'), ('and', 'CC'), ('any', 'DT'), ('other', 'JJ'), ('aussies', 'NNS'), ('with', 'IN'), ('some', 'DT'), ('solid', 'JJ'), ('things', 'NNS'), ('to', 'TO'), ('say', 'VB'), ('to', 'TO'), ('cops', 'NNS'), ('and', 'CC'), ('any', 'DT'), ('advice', 'NN'), ('in', 'IN'), ('general', 'JJ'), ('?', '.')]]

```python
# %%capture

# Inspired from https://stackoverflow.com/a/15590384
from nltk.corpus import wordnet
from nltk.stem.wordnet import WordNetLemmatizer

def get_wordnet_pos(treebank_tag):

    if treebank_tag.startswith('J'):
        return wordnet.ADJ
    elif treebank_tag.startswith('V'):
        return wordnet.VERB
    elif treebank_tag.startswith('N'):
        return wordnet.NOUN
    elif treebank_tag.startswith('R'):
        return wordnet.ADV
    else:
        return ''

lemmatizer = WordNetLemmatizer()

# Lemmatizing each word with its POS tag, in each sentence
data['tokens_sentences_lemmatized'] = data['POS_tokens'].progress_map(
    lambda list_tokens_POS: [
        [
            lemmatizer.lemmatize(el[0], get_wordnet_pos(el[1]))
            if get_wordnet_pos(el[1]) != '' else el[0] for el in tokens_POS
        ]
        for tokens_POS in list_tokens_POS
    ]
)

data['tokens_sentences_lemmatized'].head(1).tolist()[0][:3]
```

    HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=2243.0), HTML(value='')))

    [['Australia',
      'health',
      'order',
      'This',
      'thread',
      'be',
      'political',
      'as',
      'it',
      'be',
      'about',
      'current',
      'politically',
      'motivated',
      'restriction',
      'in',
      'Australia',
      '.'],
     ['Aus',
      'bros',
      'I',
      'need',
      'some',
      'help',
      'here',
      ',',
      'I',
      'hear',
      'a',
      'lot',
      'about',
      'legal',
      'restriction',
      'and',
      'confront',
      'police',
      'about',
      'their',
      'legality',
      '(',
      'I',
      "'m",
      'in',
      'Vic',
      'but',
      'whatever',
      'state',
      ')',
      '.'],
     ['Can',
      'you',
      'fellas',
      'help',
      'me',
      'and',
      'any',
      'other',
      'aussie',
      'with',
      'some',
      'solid',
      'thing',
      'to',
      'say',
      'to',
      'cop',
      'and',
      'any',
      'advice',
      'in',
      'general',
      '?']]

```python
# %%capture

from nltk.corpus import stopwords

stopwords_verbs = ['https', 'http', 'say', 'get', 'go', 'know', 'may', 'need', 'like', 'make', 'see', 'want', 'come', 'take', 'use', 'would', 'can']
stopwords_other = ['one', 'mr', 'image', 'getty', 'de', 'en', 'caption', 'also', 'copyright', 'something']
my_stopwords = stopwords.words('english') + stopwords_verbs + stopwords_other

```

```python
# %%capture

from itertools import chain # to flatten list of sentences of tokens into list of tokens
data['tokens'] = data['tokens_sentences_lemmatized'].map(lambda sentences: list(chain.from_iterable(sentences)))
data['tokens'] = data['tokens'].map(lambda tokens: [token.lower() for token in tokens if token.isalpha()
                                                    and token.lower() not in my_stopwords and len(token)>1])
data['tokens'].head(1).tolist()[0][:30]

```

    ['australia',
     'health',
     'order',
     'thread',
     'political',
     'current',
     'politically',
     'motivated',
     'restriction',
     'australia',
     'aus',
     'bros',
     'help',
     'hear',
     'lot',
     'legal',
     'restriction',
     'confront',
     'police',
     'legality',
     'vic',
     'whatever',
     'state',
     'fellas',
     'help',
     'aussie',
     'solid',
     'thing',
     'cop',
     'advice']

```python
# %%capture

from gensim.models import Phrases
from gensim import corpora

tokens = data['tokens'].tolist()
bigram_model = Phrases(tokens)
trigram_model = Phrases(bigram_model[tokens], min_count=1)
tokens = list(trigram_model[bigram_model[tokens]])

dictionary_LDA = corpora.Dictionary(tokens)
dictionary_LDA.filter_extremes(no_below=3)
corpus = [dictionary_LDA.doc2bow(tok) for tok in tokens]

```

```python
# %%capture

from gensim import models
import numpy as np

np.random.seed(123456)
num_topics = 20
%time lda_model = models.LdaModel(corpus, num_topics=num_topics, \
                                  id2word=dictionary_LDA, \
                                  passes=4, alpha=[0.01]*num_topics, \
                                  eta=[0.01]*len(dictionary_LDA.keys()))

```

    CPU times: user 9.72 s, sys: 0 ns, total: 9.72 s
    Wall time: 9.73 s

```python
# %%capture

import seaborn as sns;

topics = [lda_model[corpus[i]] for i in range(len(data))]

def topics_document_to_dataframe(topics_document, num_topics):
    res = pd.DataFrame(columns=range(num_topics))
    for topic_weight in topics_document:
        res.loc[0, topic_weight[0]] = topic_weight[1]
    return res

# Like TF-IDF, create a matrix of topic weighting, with documents as rows and topics as columns
document_topic = pd.concat([topics_document_to_dataframe(topics_document, num_topics=num_topics) for topics_document in topics]) \
  .reset_index(drop=True).fillna(0)

```

```python
# %%capture

document_topic.head()

%matplotlib inline
sns.set(rc={'figure.figsize':(10,20)})
sns.heatmap(document_topic.loc[document_topic.idxmax(axis=1).sort_values().index])
```

```python
# %%capture

sns.set(rc={'figure.figsize':(10,5)})
document_topic.idxmax(axis=1).value_counts().plot.bar(color='lightblue')
```

```python
# %%capture

# https://cran.r-project.org/web/packages/LDAvis/vignettes/details.pdf
# Here a short legend to explain the vis:
# size of bubble: proportional to the proportions of the topics across the N total tokens in the corpus
# red bars: estimated number of times a given term was generated by a given topic
# blue bars: overall frequency of each term in the corpus
# -- Relevance of words is computed with a parameter lambda
# -- Lambda optimal value ~0.6 (https://nlp.stanford.edu/events/illvi2014/papers/sievert-illvi2014.pdf)
%matplotlib inline
import pyLDAvis
import pyLDAvis.gensim
vis = pyLDAvis.gensim.prepare(topic_model=lda_model, corpus=corpus, dictionary=dictionary_LDA)
pyLDAvis.enable_notebook()
pyLDAvis.display(vis)

```

<link rel="stylesheet" type="text/css" href="https://cdn.rawgit.com/bmabey/pyLDAvis/files/ldavis.v1.0.0.css">

```python
# %%capture
from wordcloud import WordCloud,STOPWORDS

def showWordCloud(data):
    words = ' '.join(data)
    cleaned_word = " ".join([word for word in words.split()])
    wordcloud = WordCloud(stopwords = STOPWORDS,
                         background_color = 'black',
                         width = 2500,
                         height = 2500
                         ).generate(cleaned_word)
    plt.figure(1,figsize = (13,13))
    plt.imshow(wordcloud)
    plt.axis('off')
    plt.show()

flat_list = [item for sublist in tokens for item in sublist]

#flat_list
showWordCloud(flat_list)
```

# Comparison Questions

### Q1.) ***What is the archiving service 4plen coverage of 4chan and what is the quality of the coverage?***

Between the dates of **{{_from_date.strftime("%d/%m/%Y, %H:%M:%S") }}** and **{{_to_date.strftime("%d/%m/%Y, %H:%M:%S") }}** we have crawled **{{ "{:,}".format(len(match_result)) }}** threads on 4chan /pol board.

Of these, we have collected data from all three sources - 4chan scrap using OXPath, 4chan API and 4pleb API (i.e. scraped thread, posts content and metadata) on **{{full_match_len}} / {{"%.3f" % (full_match_len/len(match_result)*100)}} %** of the results.

We have found on the 4chan archiving service, 4pleb, **{{match_result.pleb.sum()}}  / {{"%.3f" % perc_match }}%** of the results.

Looking into the remainder **{{ len(match_result) - match_result.pleb.sum() }} / {{"%.3f" % perc_not_match }}%** that are not found in the archiving service we have noticed that:

- We have scraped **{{ len(not_pleb_yes_chan) }} / {{ "%.3f" % (len(not_pleb_yes_chan)/len(not_pleb_match)*100) }}%** threads from 4chan API but not found on 4pleb.
- We have scraped **{{ len(not_pleb_yes_chan_or_oxp) }} / {{ "%.3f" % (len(not_pleb_yes_chan_or_oxp)/len(not_pleb_match)*100) }}** using OXPath or 4chan API scraping methods. Of these **{{ len(not_pleb_match.query("oxp")) }}** is the number of items scraped using OXPath.

The analysis of the deleted threads could lead to insight on how threads are being monitored

- There are **{{ "{:,}".format(len(match_result.query("deleted==True"))) }}** threads that have been marked as deleted by the OXPath scraper. Of the deleted threads, **{{ "{:,}".format(len(match_result.query("pleb").query("deleted==True"))) }}** have been found been found on the 4pleb archiving API.
- There are **{{ len(id_chan_posts) }}** threads found on 4chan API that have been marked as deleted. **{{ num_del_chan }}** have also been found and marked as deleted by OXpath scraper
- There are **{{ len(pleb_posts.query("deleted=='1'")) }}** threads marked as deleted in 4pleb. **{{len(match_result[match_result['num'].isin(id_pleb_posts)].query('deleted==True'))}}** have also been found and marked as deleted by OXpath scraper

The deleted threads seems to behave according to the following pattern:

* Threads are posted by original poster (op)
* There is some activity on the thread (replies and/or reads)
* The thread becomes in-accessible and its URL return *404 - Page not found*
    * The OXPath scraper marks the thread as deleted
* Some threads are deleted permanently whilst others are returned on the main board and the activity continues
    * We can assume that **{{ "%.3f" % ( (len(pleb_posts.query("deleted=='1'")) / len(match_result.query("pleb").query("deleted==True"))) * 100)  }}%** of the cases that 4pleb also finds to be deleted, is the actual number of deleted threads

### Q1.1) - What topics of threads are being deleted and what is the difference between the ones permanently deleted and the ones put back on the board?

When analysing deleted threads we have to take into consideration the following question:
    ***How do we know when a thread has been deleted?***

The question is non trivial because ultimately ***all*** threads get deleted from 4chan. We are interested in a ***deleted*** status of the thread outside of it's expected life cycle.  The live expectancy of a thread $L_t$ (i.e. the elapsed amount of time during which a thread is reachable by it's uniquer URL) is composed of two components: $E_b$ - *Elapsed* time on the *board* and $E_a$ - *Elapsed* time *archived*.

\begin{equation} \label{eq1}
    L_t = E_b + E_a
\end{equation}

The second component $E_a$ is expected to be constant. All threads remain available for three days after they are archived. (add reference)

The first element, the elapsed time a thread spends on the board $E_b$, is an entity of time, and function of activity and janitor moderation. The activity on a thread is a combination of posts/images/authors. We can simplify this by only considering the number of posts (i.e. replies). Each new post bumps the thread up on the board, delaying it's archiving time (add reference) . The bumping effect is also a function that decays with time. Alternatively, since $E_b$ already represents time, it can be expressed based on the number of posts $p$. Similarly, the rate of decay $\beta$  grows as the number of posts on the thread grows (each posts bumps the thread up a bit, but every time less than the previous).

Furthermore, emerging from data, at times threads gets removed from the board and appear to be deleted. This is expected as 4chan has various levels of moderation by its *janitors*. Since threads can also be in a deleted state and then put back in the live board we are going to consider $P(d)$ as the probability of a threat being deleted with it's values $P(d) \in [0,1]  where  P(d)=0$ thread was not deleted, $P(d)=1$ thread was deleted, $P(d) \in (0,1)$ thread was deleted for a certain amount of time as percentage of its elapsed time and then put back in the live board.

\begin{equation}
    E_b = [log(np)(1-\frac{p}{max_p})^{log(p)}]*(1-P(d))
\end{equation}

Since we know that n = 200, and assuming $P(d)$ only affects threads in the live board

\begin{equation}
    L_t = [log(200p)(1-\frac{p}{max_p})^{log(p)}]*(1-P(d)) + E_a
\end{equation}

where

- $L_t$ Elapsed life (time) of a thread
- $n$ Number of threads on the board
- $p$ Number of posts on the thread
- $max_p$ The maximum number of posts any thread on the board has
- $\beta = p/max_p$ rate of decay of the thread through out the board
- $P(d)$ The probability that a thread has been deleted
- $E_a$ The elapsed time during which the thread is available in the archived board

Should a.) Load all threads, nr_replies, elapsed time between start date and last post, deleted flag b.) plot calculated time and observed time, c.) check difference and formula fit.

**(question outside of main quest - dropping it)**

```python
# %%capture
print('Plot showing the distribution of number of replies for missed threads)')

f2 = f1[f1.chan == True]
chan_op = pd.DataFrame(list(f2['chan_op']))
chan_op.drop(axis=1, inplace=True, columns=['md5', 'resto', 'imagelimit', 'troll_country'])#, 'm_img', 'tail_size', 'trip'])
chan_op.drop(axis=1, inplace=True, columns=['ext',	'w',	'h',	'tn_w',	'tn_h',
                      'tim',	'time',	'fsize',	'country',
                      'archived',	'bumplimit',	'semantic_url',
                      'country_name',	'images'])
chan_op = chan_op.sort_values('replies', ascending=False)
plt.hist(chan_op.replies, bins=20)

```

```python
# %%capture
import time
from datetime import datetime

print('Plot showing the time to being arcived of the threads not captured)')

time_op = pd.DataFrame([chan_op.no, chan_op.archived_on, chan_op.now])
time_op = time_op.transpose()
time_op['now'] = pd.to_datetime(time_op['now'] , format='%m/%d/%y(%a)%X')
#time_op['archived_on'] = time_op['archived_on'].apply(time.localtime)
time_op['archived_on'] = time_op['archived_on'].apply(datetime.fromtimestamp)
time_op['length'] =  (time_op.archived_on - time_op.now).astype('timedelta64[h]')
time_op = time_op.sort_values(['length'])
plt.hist(time_op.length, bins=20)

```

```python
# %%capture
import numpy as np
import matplotlib.pyplot as plt

plt.rcParams.update({'figure.figsize':(7,5), 'figure.dpi':100})

plt.hist(count, bins=50)
plt.gca().set(title='Frequency Histogram', ylabel='Frequency');

```

```python
# %%capture
pipeline = [
            {"$match": {"api_scrap": True } }, { "$addFields": {
        "pleb_array": {
                "$ifNull": [ {"$objectToArray": "$pleb"}, [ ] ]
        }
    } },
    {"$project": {"_id": 0, "url": "$_id", "api_chan_posts": { "$size": { "$ifNull": [ "$chan.posts", [ ] ] } } ,
                "api_chan_op": {"$arrayElemAt": ["$chan.posts", 0 ] }, "pleb_array": {"$arrayElemAt": ["$pleb_array", 0 ] } } },
    {"$project": {"url": 1, "api_chan_posts": 1, "api_chan_op": 1
                ,"api_pleb_posts":  {"$ifNull": [ {"$objectToArray": "$pleb_array.v.posts"}, [ ] ] }
                ,"api_pleb_op": "$pleb_array.v.op"  } } ,
     {"$project": {"url": 1, "api_chan_posts": 1, "api_chan_op": 1, "api_pleb_op": 1, "api_pleb_posts": {"$size": "$api_pleb_posts"} } }
]

thread_data = pd.DataFrame(threads.aggregate(pipeline, allowDiskUse=True))

```

```python
# %%capture
pipeline = [
    {"$match": {"$and": [{ "data": {"$ne": None } },{"data": {"$type": "array" } } ] } }
   ,{"$project": { "url": 1, "chan_posts": {"$arrayElemAt": ["$data", -1 ] } } }
   ,{"$project": { "url": 1, "chan_posts": {"$arrayElemAt": ["$chan_posts.payload.thread", 0] } } }
   ,{"$project": { "_id":0, "url": 1, "chan_posts": {"$size": {"$ifNull": [ "$chan_posts.replies", [ ] ] } },
       "chan_op": "$chan_posts.op" } }
   ,{"$unwind": "$chan_op"}
]

original_thread_data = pd.DataFrame(livechan.aggregate(pipeline, allowDiskUse=True))
len(original_thread_data)
# sum(original_thread_data['chan_posts'])

```

```python
# %%capture
import json
#plt.plot(thread_data['api_chan_posts'] - thread_data['api_bleb_posts'])
merged_data = pd.merge(thread_data, original_thread_data, on=["url"])
merged_data['diff_posts'] = abs(merged_data['api_chan_posts'] - merged_data['api_pleb_posts'])
merged_data = merged_data.sort_values(by=['diff_posts'], ascending=False)
print(f'OXpath Threads {len(original_thread_data)}')
print(f'4Pleb and 4 Chan Threads {len(thread_data)}')
print(f'Merged match {len(merged_data)}')

# merged_data.api_chan_op[merged_data.api_chan_op == None]['api_chan_op'] = { }
# merged_data.api_chan_op1 = pd.DataFrame(list(merged_data.api_chan_op))

```

```python
# %%capture
import numpy
import scipy.stats as ss
no_api_chan = merged_data[~merged_data.api_chan_op.isnull()].sort_values(by=['diff_posts'], ascending=False)
no_api_pleb = no_api_chan[~no_api_chan.api_pleb_op.isnull()].sort_values(by=['diff_posts'], ascending=False)
#no_api_pleb = no_api_pleb[no_api_pleb.diff_posts > 0]
#print(no_api_pleb.diff_posts.mean())
#print(no_api_pleb.diff_posts.std())
#ss.describe(no_api_pleb.diff_posts)
#diff_pdf = ss.norm.pdf(no_api_pleb.diff_posts, 0, 1)
#plt.plot(no_api_pleb.diff_posts, diff_pdf, label='pdf')

mu = 0
sigma = 1
diff_cdf = ss.norm.cdf(no_api_pleb.diff_posts, mu, sigma)
diff_pdf = ss.norm.pdf(no_api_pleb.diff_posts, mu, sigma)

fig=plt.figure()
ax=fig.add_subplot(111)
ax.plot(no_api_pleb.diff_posts, diff_pdf, label='pdf')
ax.plot(no_api_pleb.diff_posts, diff_cdf, label='cdf')
ax.set_xlim(xmin=0.0, xmax=10)
ax.legend()
plt.show()

```

### Q2.) What is the persentage of posts that can be matched on all three crawling methods and what are the gaps

Whilst looking at the thread coverage from the archiving sites, we can see promising results with matching results well above $95.00$% another question is around the quality of the coverage. That is, do the threads discovered and retrieved from the archiving services (in this case 4pleb API) have the same information as what retrieved from 4chan API and/or OXPath scraping method?

The best way to look at quality would be to:
1. verify that the thread contains and matches some of the key metadata (such as date it was posted, date it was archived, author country)
2. verify that the thread contains and matches its main content (such as subject, comment, media) and lastly
3. verify that the linked posts match with what is reported from 4chan API and/or OXPath scraping method.

### Q3.) What can be said about the significat amount of threads that get flagged/marked as "deleted/404" and either permanently deleted or returned to the board. What topics get permanently deleted? What topics get flagged?


#Imports

```
import numpy as np
%matplotlib inline
import numpy as np
import matplotlib.pyplot as plt
from IPython import display
plt.style.use('seaborn-white')
```

# Read and process data.

Download the file from this URL: https://drive.google.com/file/d/1UWWIi-sz9g0x3LFvkIZjvK1r2ZaCqgGS/view?usp=sharing

```
data = open('text.txt', 'r').read()
```

```
data
```

    'A coronavirus is a kind of common virus that causes an infection in your nose, sinuses, or upper throat. Most coronaviruses are not dangerous.\n\nSome types of coronaviruses are serious, though. About 858 people have died from Middle East respiratory syndrome (MERS), which first appeared in 2012 in Saudi Arabia and then in other countries in the Middle East, Africa, Asia, and Europe. In April 2014, the first American was hospitalized for MERS in Indiana and another case was reported in Florida. Both had just returned from Saudi Arabia. In May 2015, there was an outbreak of MERS in Korea, which was the largest outbreak outside of the Arabian Peninsula. In 2003, 774 people died from a severe acute respiratory syndrome (SARS) outbreak. As of 2015, there were no further reports of cases of SARS. \n\nBut in early 2020, following a December 2019 outbreak in China, the World Health Organization identified a new type, 2019 novel coronavirus (2019-nCoV).\n\nOften a coronavirus causes upper respiratory infection symptoms like a stuffy nose, cough, and sore throat. You can treat them with rest and over-the-counter medication. The coronavirus can also cause middle ear infections in children.\n\nWhat Is a Coronavirus?\nCoronaviruses were first identified in the 1960s, but we don\'t know where they come from. They get their name from their crown-like shape. Sometimes, but not often, a coronavirus can infect both animals and humans.\n\nMost coronaviruses spread the same way other cold-causing viruses do: through infected people coughing and sneezing, by touching an infected person\'s hands or face, or by touching things such as doorknobs that infected people have touched.\n\nAlmost everyone gets a coronavirus infection at least once in their life, most likely as a young child. In the United States, coronaviruses are more common in the fall and winter, but anyone can come down with a coronavirus infection at any time.\n\nCommon Symptoms of Coronavirus\nThe symptoms of most coronaviruses are similar to any other upper respiratory infection, including runny nose, coughing, sore throat, and sometimes a fever. In most cases, you won\'t know whether you have a coronavirus or a different cold-causing virus, such as rhinovirus.\n\nYou could get lab tests, including nose and throat cultures and blood work, to find out whether your cold was caused by a coronavirus, but there\'s no reason to. The test results wouldn\'t change how you treat your symptoms, which typically go away in a few days.\n\nBut if a coronavirus infection spreads to the lower respiratory tract (your windpipe and your lungs), it can cause pneumonia, especially in older people, people with heart disease, or people with weakened immune systems.\n\nWhat to Do About Coronavirus\nThere is no vaccine for coronavirus. To help prevent a coronavirus infection, do the same things you do to avoid the common cold:\n\nWash your hands thoroughly with soap and warm water or with an alcohol-based hand sanitizer.\nKeep your hands and fingers away from your eyes, nose, and mouth.\nAvoid close contact with people who are infected.\nYou treat a coronavirus infection the same way you treat a cold:\n\nGet plenty of rest.\nDrink fluids.\nTake over-the-counter medicine for a sore throat and fever. But don\'t give aspirin to children or teens younger than 19; use ibuprofen or acetaminophen instead.\nA humidifier or steamy shower can also help ease a sore and scratchy throat.\n\nEven when a coronavirus causes MERS or SARS in other countries, the kind of coronavirus infection common in the U.S. isn\'t a serious threat for an otherwise healthy adult. If you get sick, treat your symptoms and contact a doctor if they get worse or don\'t go away.\n\nNews about the coronavirus outbreak that started in Wuhan, China, is changing rapidly. The respiratory infection, which is closely related to SARS and MERS, has been spreading across China, and cases have been diagnosed in several other countries, including the United States. We’ll provide the latest updates on cases, deaths, travel restrictions, and more here.\n\nWhat is the latest news?\nOn Wednesday, CDC medical officers and others met a group of about 210 U.S. citizens evacuated from China. Their plane landed at March Air Reserve Base in California, where the evacuees will be monitored for coronavirus symptoms for several days. Anyone showing signs of the disease will be taken to the hospital.\n\nSome airlines, including British Airways and United Airlines, are stopping or reducing flights to China as the number of cases continues to rise rapidly. Worldwide there are now more than 6,000 cases and 132 deaths, according to the European Centre for Disease Prevention and Control. While the majority of cases center in China, it has been found in the U.S. and these countries: Thailand, Hong Kong, Taiwan, Australia, Macau, Singapore, Japan, South Korea, Malaysia, France, Canada, Vietnam, Nepal, Cambodia, Germany and United Arab Emirates. There have been no deaths outside China.\n\nIn the U.S., the CDC is reporting five confirmed cases from these states: Washington, Illinois, California and Arizona. On Monday, Jan. 27, Nancy Messonnier, MD, Director of the agency\'s National Center for Immunization and Respiratory Diseases, said it had 165 persons under investigation for coronavirus from 36 states. In addition to the 5 confirmed positive, 68 have tested negative. They are prioritizing the testing based on a person’s risk.\n\nMessonnier said they had posted the blueprints for their diagnostic test on a public server and were working "as fast as we can" to get test kits out to states. \n\nRight now, all the testing for the new coronavirus is taking place at the CDC\'s headquarters in Atlanta.\n\nShe said the CDC was mulling a change to its travel screening for the infection, but did not give further details.\n\nHow many people have been diagnosed with the virus, and how many have died?\nAccording to European CDC, the majority of the confirmed cases – 5,994 – are in China. Another 78 cases are confirmed outside of China in 16 countries. Countries with the most confirmed cases include Thailand with 14 and Taiwan and South Korea with 8 each. All reported deaths have been in China, and include 16 healthcare workers.\n\nWhen did the outbreak start?\nChina first reported the outbreak in Wuhan on Dec. 30, 2019.\n\nWhat are public officials doing to contain the virus?\nOn Tuesday, the CDC urged travelers to avoid non-essential travel to China.\n\nChinese officials have shut down all public transportation to 10 cities, affecting 35 million people. The first was Wuhan, which has a population of about 11 million. In Wuhan, that includes buses, subways, trains, and the airport.\n\nAll passengers flying into the United States from Wuhan will be routed through one of five airports and screened for fever and other signs of the virus. “Patients of concern��� will be referred to a facility and given a test to find out if they have the virus.\n\nWhat do we know about cases in the United States?\nIn all U.S. cases so far, patients had recently traveled to Wuhan.\n\nCalifornia has two patients, one in Los Angeles County and one in Orange County. The patient from Orange County is a man in his 50s. He is in a local hospital in isolation and is in good condition, according to the Orange County Health Care Agency. Los Angeles County officials did not provide additional details about the patient there.\n\nArizona\'s Department of Health Services said its patient is a Maricopa County resident and member of the Arizona State University community who did not live in student housing. The patient is not severely ill and is being kept in isolation. \n\nAnother case involves a woman in her 60s from Chicago. The Chicago Department of Public Health reported that she had visited Wuhan, China in December and returned to Chicago earlier this month. She is hospitalized in stable condition. \n\nThe first U.S. patient is a man in his 30s from Washington state. He had traveled from Wuhan and entered the country before the screening was in place. He started having symptoms and contacted his doctor. He is in good condition and is in isolation at Providence Regional Medical Center.\n\nIs travel to China safe?\nWuhan is closed to travelers.\n\nThe CDC is advising that travelers avoid non-essential travel to China. Travelers who do go should:\n\nAvoid contact with sick people.\nAvoid animals, animal markets, and products that come from animals.\nWash their hands often with soap and water, or use an alcohol-based sanitizer if that’s not available.\nSeek medical care right away for a fever, cough, or difficulty breathing. Tell a health care professional about any travel.\nWhat are the symptoms, and how is the virus diagnosed?\nChina created a test for the virus and shared that information with other countries. The CDC has developed its own test.\n\nSymptoms include a fever, coughing, and shortness of breath. They may appear 2 to 14 days after you’re exposed to the virus.\n\nWhat is the source of the virus, and how is it spread?\nHealth officials are not sure of the source of the virus yet or how easily it can spread. Coronaviruses are found in many different animals, including camels, cattle, cats, and bats. One research paper also suggested snakes as a possible source. The new virus may be linked to a seafood and live animal market in Wuhan that has since been closed\n\nThe virus can spread from person to person. Health officials are seeing this happen most often where people are close together and in health care settings. To date, 16 health care workers have been infected.\n\nThe CDC believes that severe acute respiratory syndrome (SARS) and Middle East respiratory syndrome (MERS), two other types of coronavirus, are spread through droplets when someone coughs or sneezes.\n\nIs there a vaccine?\nThere is no vaccine, but the National Institutes of Health is working on one and hopes to begin testing in several months. That testing would be for safety. If it’s safe, there would be testing to see how well it works.\n\nHow is it treated?\nThere is no specific treatment for the virus. Patients are generally given supportive care for their symptoms, such a fluids and pain relievers. Hospitalized patients may need support with breathing.'

Process data and calculate indices

```
chars = list(set(data))
print(chars)
data_size, X_size = len(data), len(chars)
print("Corona Virus article has %d characters, %d unique characters" %(data_size, X_size))
char_to_idx = {ch:i for i,ch in enumerate(chars)}
idx_to_char = {i:ch for i,ch in enumerate(chars)}
print(char_to_idx)
print(idx_to_char)
```

    ['V', '3', '–', "'", ')', 'J', 'g', 'f', 'K', '8', 'Y', 'r', 'H', '6', 'W', 'a', '’', 'i', 'S', '0', 'y', 'z', 'o', 'c', '2', 'R', 'm', 'P', 'O', 'I', 'l', 'N', '1', 'M', '4', ':', 'q', '“', 'L', 'F', 'A', '�', 'C', 'j', 't', 'b', ',', 'E', 'e', 'B', 'x', 's', '-', '"', 'w', '7', 'D', 'v', 'U', 'T', '?', ' ', '.', 'G', 'd', 'k', '5', ';', '(', 'n', '\n', '9', 'h', 'u', 'p']
    Corona Virus article has 10223 characters, 75 unique characters
    {'V': 0, '3': 1, '–': 2, "'": 3, ')': 4, 'J': 5, 'g': 6, 'f': 7, 'K': 8, '8': 9, 'Y': 10, 'r': 11, 'H': 12, '6': 13, 'W': 14, 'a': 15, '’': 16, 'i': 17, 'S': 18, '0': 19, 'y': 20, 'z': 21, 'o': 22, 'c': 23, '2': 24, 'R': 25, 'm': 26, 'P': 27, 'O': 28, 'I': 29, 'l': 30, 'N': 31, '1': 32, 'M': 33, '4': 34, ':': 35, 'q': 36, '“': 37, 'L': 38, 'F': 39, 'A': 40, '�': 41, 'C': 42, 'j': 43, 't': 44, 'b': 45, ',': 46, 'E': 47, 'e': 48, 'B': 49, 'x': 50, 's': 51, '-': 52, '"': 53, 'w': 54, '7': 55, 'D': 56, 'v': 57, 'U': 58, 'T': 59, '?': 60, ' ': 61, '.': 62, 'G': 63, 'd': 64, 'k': 65, '5': 66, ';': 67, '(': 68, 'n': 69, '\n': 70, '9': 71, 'h': 72, 'u': 73, 'p': 74}
    {0: 'V', 1: '3', 2: '–', 3: "'", 4: ')', 5: 'J', 6: 'g', 7: 'f', 8: 'K', 9: '8', 10: 'Y', 11: 'r', 12: 'H', 13: '6', 14: 'W', 15: 'a', 16: '’', 17: 'i', 18: 'S', 19: '0', 20: 'y', 21: 'z', 22: 'o', 23: 'c', 24: '2', 25: 'R', 26: 'm', 27: 'P', 28: 'O', 29: 'I', 30: 'l', 31: 'N', 32: '1', 33: 'M', 34: '4', 35: ':', 36: 'q', 37: '“', 38: 'L', 39: 'F', 40: 'A', 41: '�', 42: 'C', 43: 'j', 44: 't', 45: 'b', 46: ',', 47: 'E', 48: 'e', 49: 'B', 50: 'x', 51: 's', 52: '-', 53: '"', 54: 'w', 55: '7', 56: 'D', 57: 'v', 58: 'U', 59: 'T', 60: '?', 61: ' ', 62: '.', 63: 'G', 64: 'd', 65: 'k', 66: '5', 67: ';', 68: '(', 69: 'n', 70: '\n', 71: '9', 72: 'h', 73: 'u', 74: 'p'}

# Constants and Hyperparameters

```
Hidden_Layer_size = 100 #size of the hidden layer
Time_steps = 40 # Number of time steps (length of the sequence) used for training
learning_rate = 1e-1 # Learning Rate
weight_sd = 0.1 #Standard deviation of weights for initialization
z_size = Hidden_Layer_size + X_size #Size of concatenation(H, X) vector
```

```
z_size
```

    175

# Activation Functions and Derivatives

```
def sigmoid(x):
    return np.exp(x) / (1 + np.exp(x))

def dsigmoid(x):
    return x * (1 - x)

def tanh(x):
    x = np.asarray(x)
    return (np.exp(2*x) - 1) / (np.exp(2 * x) + 1)

def dtanh(x):
    return 1 - x * x
```

# Quiz Question 1

What is the value of sigmoid(0) calculated from  your code? (Answer up to 1 decimal point, e.g. 4.2 and NOT 4.29999999, no rounding off).

# Quiz Question 2

What is the value of dsigmoid(sigmoid(0)) calculated from your code?? (Answer up to 2 decimal point, e.g. 4.29 and NOT 4.29999999, no rounding off).

# Quiz Question 3

What is the value of tanh(dsigmoid(sigmoid(0))) calculated from your code?? (Answer up to 5 decimal point, e.g. 4.29999 and NOT 4.29999999, no rounding off).

# Quiz Question 4

What is the value of dtanh(tanh(dsigmoid(sigmoid(0)))) calculated from your code?? (Answer up to 5 decimal point, e.g. 4.29999 and NOT 4.29999999, no rounding off).

```
# Q1
sigmoid(0)
```

    0.5

```
# Q2
dsigmoid(sigmoid(0))
```

    0.25

```
# Q3
tanh(dsigmoid(sigmoid(0)))
```

    0.24491866240370913

```
# Q4
dtanh(tanh(dsigmoid(sigmoid(0))))
```

    0.940014848806378

# Parameters

```
class Param:
    def __init__(self, name, value):
      self.name = name
      self.v = value # parameter value
      self.d = np.zeros_like(value) # derivative
      self.m = np.zeros_like(value) # momentum for Adagrad
```

We use random weights with normal distribution (0, weight_sd) for  tanh  activation function and (0.5, weight_sd) for  `sigmoid`  activation function.

Biases are initialized to zeros.

# LSTM
You are making this network, please note f, i, c and o (also "v") in the image below:

Please note that we are concatenating the old_hidden_vector and new_input.

# Quiz Question 4

In the class definition below, what should be size_a, size_b, and size_c? ONLY use the variables defined above.

```
size_a = Hidden_Layer_size# write your code here
size_b = z_size# write your code here
size_c = X_size# write your code here

class Parameters:
    def __init__(self):
        self.W_f = Param('W_f', np.random.randn(size_a, size_b) * weight_sd + 0.5)
        self.b_f = Param('b_f', np.zeros((size_a, 1)))

        self.W_i = Param('W_i', np.random.randn(size_a, size_b) * weight_sd + 0.5)
        self.b_i = Param('b_i', np.zeros((size_a, 1)))

        self.W_C = Param('W_C', np.random.randn(size_a, size_b) * weight_sd)
        self.b_C = Param('b_C', np.zeros((size_a, 1)))

        self.W_o = Param('W_o', np.random.randn(size_a, size_b) * weight_sd + 0.5)
        self.b_o = Param('b_o', np.zeros((size_a, 1)))

        #For final layer to predict the next character
        self.W_v = Param('W_v', np.random.randn(X_size, size_a) * weight_sd)
        self.b_v = Param('b_v', np.zeros((size_c, 1)))

    def all(self):
        return [self.W_f, self.W_i, self.W_C, self.W_o, self.W_v,
               self.b_f, self.b_i, self.b_C, self.b_o, self.b_v]

parameters = Parameters()
```

```
parameters.all()
```

    [<__main__.Param at 0x7f9fd7685cc0>,
     <__main__.Param at 0x7f9fd767f550>,
     <__main__.Param at 0x7f9fd767f5c0>,
     <__main__.Param at 0x7f9fd767f630>,
     <__main__.Param at 0x7f9fd767f6a0>,
     <__main__.Param at 0x7f9fd767f4a8>,
     <__main__.Param at 0x7f9fd767f588>,
     <__main__.Param at 0x7f9fd767f5f8>,
     <__main__.Param at 0x7f9fd767f668>,
     <__main__.Param at 0x7f9fd767f6d8>]

Look at these operations which we'll be writing:

**Concatenation of h and x:**

$z\:=\:\left[h_{t-1},\:x\right]$

$f_t=\sigma\left(W_f\cdot z\:+\:b_f\:\right)$

$i_i=\sigma\left(W_i\cdot z\:+\:b_i\right)$

$\overline{C_t}=\tanh\left(W_C\cdot z\:+\:b_C\right)$

$C_t=f_t\ast C_{t-1}+i_t\ast \overline{C}_t$

$o_t=\sigma\left(W_o\cdot z\:+\:b_i\right)$

$h_t=o_t\ast\tanh\left(C_t\right)$

**Logits:**

$v_t=W_v\cdot h_t+b_v$

**Softmax:**

$\hat{y}=softmax\left(v_t\right)$

```
param_dict = {x.name: x.v for x in parameters.all()}
param_dict.keys()
```

    dict_keys(['W_f', 'W_i', 'W_C', 'W_o', 'W_v', 'b_f', 'b_i', 'b_C', 'b_o', 'b_v'])

```
def forward(x, h_prev, C_prev, p = parameters):
    assert x.shape == (X_size, 1)
    assert h_prev.shape == (Hidden_Layer_size, 1)
    assert C_prev.shape == (Hidden_Layer_size, 1)
    param_dict = {x.name: x.v for x in parameters.all()}
    W_f = param_dict['W_f']
    W_i = param_dict['W_i']
    W_C = param_dict['W_C']
    b_f = param_dict['b_f']
    b_i = param_dict['b_i']
    b_C = param_dict['b_C']
    W_o = param_dict['W_o']
    b_o = param_dict['b_o']
    W_v = param_dict['W_v']
    b_v  = param_dict['b_v']
    z = np.row_stack((h_prev, x))
    f = sigmoid(np.dot(W_f, z) + b_f)# write your code here
    i = sigmoid(np.dot(W_i, z) + b_i) # write your code here
    C_bar =tanh(np.dot(W_C, z) + b_C)# write your code here

    C = f * C_prev + i * C_bar# write your code here
    o =  sigmoid(np.dot(W_o, z) + b_o )# write your code here
    h = o * tanh(C)# write your code here

    v = np.dot(W_v, h) + b_v# write your code here
    y = np.exp(v) / (np.sum(np.exp(v)) + 1e-8) #softmax

    return z, f, i, C_bar, C, o, h, v, y
```

You must finish the function above before you can attempt the questions below.

# Quiz Question 5

What is the output of 'print(len(forward(np.zeros((X_size, 1)), np.zeros((Hidden_Layer_size, 1)), np.zeros((Hidden_Layer_size, 1)), parameters)))'?

```
print(len(forward(np.zeros((X_size, 1)), np.zeros((Hidden_Layer_size, 1)), np.zeros((Hidden_Layer_size, 1)), parameters)))
```

    9

# Quiz Question 6.

Assuming you have fixed the forward function, run this command:
z, f, i, C_bar, C, o, h, v, y = forward(np.zeros((X_size, 1)), np.zeros((Hidden_Layer_size, 1)), np.zeros((Hidden_Layer_size, 1)))

Now, find these values:

1.   print(z.shape)
2.   print(np.sum(z))
3.   print(np.sum(f))

Copy and paste exact values you get in the logs into the quiz.

```
print(z.shape)
print(np.sum(z))
print(np.sum(f))
```

    (175, 1)
    0.0
    50.0

```
z, f, i, C_bar, C, o, h, v, y = forward(np.zeros((X_size, 1)), np.zeros((Hidden_Layer_size, 1)), np.zeros((Hidden_Layer_size, 1)))
```

# Backpropagation

Here we are defining the backpropagation. It's too complicated, here is the whole code. (Please note that this would work only if your earlier code is perfect).

```
def backward(target, dh_next, dC_next, C_prev,
             z, f, i, C_bar, C, o, h, v, y,
             p = parameters):

    assert z.shape == (X_size + Hidden_Layer_size, 1)
    assert v.shape == (X_size, 1)
    assert y.shape == (X_size, 1)

    for param in [dh_next, dC_next, C_prev, f, i, C_bar, C, o, h]:
        assert param.shape == (Hidden_Layer_size, 1)

    dv = np.copy(y)
    dv[target] -= 1

    p.W_v.d += np.dot(dv, h.T)
    p.b_v.d += dv

    dh = np.dot(p.W_v.v.T, dv)
    dh += dh_next
    do = dh * tanh(C)
    do = dsigmoid(o) * do
    p.W_o.d += np.dot(do, z.T)
    p.b_o.d += do

    dC = np.copy(dC_next)
    dC += dh * o * dtanh(tanh(C))
    dC_bar = dC * i
    dC_bar = dtanh(C_bar) * dC_bar
    p.W_C.d += np.dot(dC_bar, z.T)
    p.b_C.d += dC_bar

    di = dC * C_bar
    di = dsigmoid(i) * di
    p.W_i.d += np.dot(di, z.T)
    p.b_i.d += di

    df = dC * C_prev
    df = dsigmoid(f) * df
    p.W_f.d += np.dot(df, z.T)
    p.b_f.d += df

    dz = (np.dot(p.W_f.v.T, df)
         + np.dot(p.W_i.v.T, di)
         + np.dot(p.W_C.v.T, dC_bar)
         + np.dot(p.W_o.v.T, do))
    dh_prev = dz[:Hidden_Layer_size, :]
    dC_prev = f * dC

    return dh_prev, dC_prev
```

# Forward and Backward Combined Pass

Let's first clear the gradients before each backward pass

```
def clear_gradients(params = parameters):
    for p in params.all():
        p.d.fill(0)
```

Clip gradients to mitigate exploding gradients

```
def clip_gradients(params = parameters):
    for p in params.all():
        np.clip(p.d, -1, 1, out=p.d)
```

Calculate and store the values in forward pass. Accumulate gradients in backward pass and clip gradients to avoid exploding gradients.

input, target are list of integers, with character indexes.
h_prev is the array of initial h at  h−1  (size H x 1)
C_prev is the array of initial C at  C−1  (size H x 1)
Returns loss, final  hT  and  CT

```
def forward_backward(inputs, targets, h_prev, C_prev):
    global paramters

    # To store the values for each time step
    x_s, z_s, f_s, i_s,  = {}, {}, {}, {}
    C_bar_s, C_s, o_s, h_s = {}, {}, {}, {}
    v_s, y_s =  {}, {}

    # Values at t - 1
    h_s[-1] = np.copy(h_prev)
    C_s[-1] = np.copy(C_prev)

    loss = 0
    # Loop through time steps
    assert len(inputs) == Time_steps
    for t in range(len(inputs)):
        x_s[t] = np.zeros((X_size, 1))
        x_s[t][inputs[t]] = 1 # Input character

        (z_s[t], f_s[t], i_s[t],
        C_bar_s[t], C_s[t], o_s[t], h_s[t],
        v_s[t], y_s[t]) = \
            forward(x_s[t], h_s[t - 1], C_s[t - 1]) # Forward pass

        loss += -np.log(y_s[t][targets[t], 0]) # Loss for at t

    clear_gradients()

    dh_next = np.zeros_like(h_s[0]) #dh from the next character
    dC_next = np.zeros_like(C_s[0]) #dh from the next character

    for t in reversed(range(len(inputs))):
        # Backward pass
        dh_next, dC_next = \
            backward(target = targets[t], dh_next = dh_next,
                     dC_next = dC_next, C_prev = C_s[t-1],
                     z = z_s[t], f = f_s[t], i = i_s[t], C_bar = C_bar_s[t],
                     C = C_s[t], o = o_s[t], h = h_s[t], v = v_s[t],
                     y = y_s[t])

    clip_gradients()

    return loss, h_s[len(inputs) - 1], C_s[len(inputs) - 1]
```

# Sample the next character

```
def sample(h_prev, C_prev, first_char_idx, sentence_length):
    x = np.zeros((X_size, 1))
    x[first_char_idx] = 1

    h = h_prev
    C = C_prev

    indexes = []

    for t in range(sentence_length):
        _, _, _, _, C, _, h, _, p = forward(x, h, C)
        idx = np.random.choice(range(X_size), p=p.ravel())
        x = np.zeros((X_size, 1))
        x[idx] = 1
        indexes.append(idx)

    return indexes
```

# Training (Adagrad)

Update the graph and display a sample output

```
def update_status(inputs, h_prev, C_prev):
    #initialized later
    global plot_iter, plot_loss
    global smooth_loss

    # Get predictions for 200 letters with current model

    sample_idx = sample(h_prev, C_prev, inputs[0], 200)
    txt = ''.join(idx_to_char[idx] for idx in sample_idx)

    # Clear and plot
    plt.plot(plot_iter, plot_loss)
    display.clear_output(wait=True)
    plt.show()

    #Print prediction and loss
    print("----\n %s \n----" % (txt, ))
    print("iter %d, loss %f" % (iteration, smooth_loss))
```

# Update Parameters

\begin{align}
\theta_i &= \theta_i - \eta\frac{d\theta_i}{\sum dw_{\tau}^2} \\
d\theta_i &= \frac{\partial L}{\partial \theta_i}
\end{align}

```
def update_paramters(params = parameters):
    for p in params.all():
        p.m += p.d * p.d # Calculate sum of gradients
        #print(learning_rate * dparam)
        p.v += -(learning_rate * p.d / np.sqrt(p.m + 1e-8))
```

To delay the keyboard interrupt to prevent the training from stopping in the middle of an iteration

```
# Exponential average of loss
# Initialize to a error of a random model
smooth_loss = -np.log(1.0 / X_size) * Time_steps

iteration, pointer = 0, 0

# For the graph
plot_iter = np.zeros((0))
plot_loss = np.zeros((0))
```

# Training Loop

```
iter = 50000
while iter > 0:
  # Reset
  if pointer + Time_steps >= len(data) or iteration == 0:
      g_h_prev = np.zeros((Hidden_Layer_size, 1))
      g_C_prev = np.zeros((Hidden_Layer_size, 1))
      pointer = 0

  inputs = ([char_to_idx[ch]
              for ch in data[pointer: pointer + Time_steps]])
  targets = ([char_to_idx[ch]
              for ch in data[pointer + 1: pointer + Time_steps + 1]])

  loss, g_h_prev, g_C_prev = \
      forward_backward(inputs, targets, g_h_prev, g_C_prev)
  smooth_loss = smooth_loss * 0.999 + loss * 0.001

  # Print every hundred steps
  if iteration % 100 == 0:
      update_status(inputs, g_h_prev, g_C_prev)

  update_paramters()

  plot_iter = np.append(plot_iter, [iteration])
  plot_loss = np.append(plot_loss, [loss])

  pointer += Time_steps
  iteration += 1
  iter = iter -1
```

# Quiz Question 7.

Run the above code for 50000 iterations making sure that you have 100 hidden layers and time_steps is 40. What is the loss value you're seeing?

6.42685


#Imports

```
import numpy as np
%matplotlib inline
import numpy as np
import matplotlib.pyplot as plt
import gdown
from IPython import display
plt.style.use('seaborn-white')
```

# Read and process data.

Download the file from this URL: https://drive.google.com/file/d/1UWWIi-sz9g0x3LFvkIZjvK1r2ZaCqgGS/view?usp=sharing

```
url = 'https://drive.google.com/uc?id=1UWWIi-sz9g0x3LFvkIZjvK1r2ZaCqgGS'
output = 'text.txt'
gdown.download(url, output, quiet=False)
```

    Downloading...
    From: https://drive.google.com/uc?id=1UWWIi-sz9g0x3LFvkIZjvK1r2ZaCqgGS
    To: /content/text.txt
    100%|██████████| 10.3k/10.3k [00:00<00:00, 2.14MB/s]

    'text.txt'

```
data = open('text.txt', 'r').read()
data
```

    'A coronavirus is a kind of common virus that causes an infection in your nose, sinuses, or upper throat. Most coronaviruses are not dangerous.\n\nSome types of coronaviruses are serious, though. About 858 people have died from Middle East respiratory syndrome (MERS), which first appeared in 2012 in Saudi Arabia and then in other countries in the Middle East, Africa, Asia, and Europe. In April 2014, the first American was hospitalized for MERS in Indiana and another case was reported in Florida. Both had just returned from Saudi Arabia. In May 2015, there was an outbreak of MERS in Korea, which was the largest outbreak outside of the Arabian Peninsula. In 2003, 774 people died from a severe acute respiratory syndrome (SARS) outbreak. As of 2015, there were no further reports of cases of SARS. \n\nBut in early 2020, following a December 2019 outbreak in China, the World Health Organization identified a new type, 2019 novel coronavirus (2019-nCoV).\n\nOften a coronavirus causes upper respiratory infection symptoms like a stuffy nose, cough, and sore throat. You can treat them with rest and over-the-counter medication. The coronavirus can also cause middle ear infections in children.\n\nWhat Is a Coronavirus?\nCoronaviruses were first identified in the 1960s, but we don\'t know where they come from. They get their name from their crown-like shape. Sometimes, but not often, a coronavirus can infect both animals and humans.\n\nMost coronaviruses spread the same way other cold-causing viruses do: through infected people coughing and sneezing, by touching an infected person\'s hands or face, or by touching things such as doorknobs that infected people have touched.\n\nAlmost everyone gets a coronavirus infection at least once in their life, most likely as a young child. In the United States, coronaviruses are more common in the fall and winter, but anyone can come down with a coronavirus infection at any time.\n\nCommon Symptoms of Coronavirus\nThe symptoms of most coronaviruses are similar to any other upper respiratory infection, including runny nose, coughing, sore throat, and sometimes a fever. In most cases, you won\'t know whether you have a coronavirus or a different cold-causing virus, such as rhinovirus.\n\nYou could get lab tests, including nose and throat cultures and blood work, to find out whether your cold was caused by a coronavirus, but there\'s no reason to. The test results wouldn\'t change how you treat your symptoms, which typically go away in a few days.\n\nBut if a coronavirus infection spreads to the lower respiratory tract (your windpipe and your lungs), it can cause pneumonia, especially in older people, people with heart disease, or people with weakened immune systems.\n\nWhat to Do About Coronavirus\nThere is no vaccine for coronavirus. To help prevent a coronavirus infection, do the same things you do to avoid the common cold:\n\nWash your hands thoroughly with soap and warm water or with an alcohol-based hand sanitizer.\nKeep your hands and fingers away from your eyes, nose, and mouth.\nAvoid close contact with people who are infected.\nYou treat a coronavirus infection the same way you treat a cold:\n\nGet plenty of rest.\nDrink fluids.\nTake over-the-counter medicine for a sore throat and fever. But don\'t give aspirin to children or teens younger than 19; use ibuprofen or acetaminophen instead.\nA humidifier or steamy shower can also help ease a sore and scratchy throat.\n\nEven when a coronavirus causes MERS or SARS in other countries, the kind of coronavirus infection common in the U.S. isn\'t a serious threat for an otherwise healthy adult. If you get sick, treat your symptoms and contact a doctor if they get worse or don\'t go away.\n\nNews about the coronavirus outbreak that started in Wuhan, China, is changing rapidly. The respiratory infection, which is closely related to SARS and MERS, has been spreading across China, and cases have been diagnosed in several other countries, including the United States. We’ll provide the latest updates on cases, deaths, travel restrictions, and more here.\n\nWhat is the latest news?\nOn Wednesday, CDC medical officers and others met a group of about 210 U.S. citizens evacuated from China. Their plane landed at March Air Reserve Base in California, where the evacuees will be monitored for coronavirus symptoms for several days. Anyone showing signs of the disease will be taken to the hospital.\n\nSome airlines, including British Airways and United Airlines, are stopping or reducing flights to China as the number of cases continues to rise rapidly. Worldwide there are now more than 6,000 cases and 132 deaths, according to the European Centre for Disease Prevention and Control. While the majority of cases center in China, it has been found in the U.S. and these countries: Thailand, Hong Kong, Taiwan, Australia, Macau, Singapore, Japan, South Korea, Malaysia, France, Canada, Vietnam, Nepal, Cambodia, Germany and United Arab Emirates. There have been no deaths outside China.\n\nIn the U.S., the CDC is reporting five confirmed cases from these states: Washington, Illinois, California and Arizona. On Monday, Jan. 27, Nancy Messonnier, MD, Director of the agency\'s National Center for Immunization and Respiratory Diseases, said it had 165 persons under investigation for coronavirus from 36 states. In addition to the 5 confirmed positive, 68 have tested negative. They are prioritizing the testing based on a person’s risk.\n\nMessonnier said they had posted the blueprints for their diagnostic test on a public server and were working "as fast as we can" to get test kits out to states. \n\nRight now, all the testing for the new coronavirus is taking place at the CDC\'s headquarters in Atlanta.\n\nShe said the CDC was mulling a change to its travel screening for the infection, but did not give further details.\n\nHow many people have been diagnosed with the virus, and how many have died?\nAccording to European CDC, the majority of the confirmed cases – 5,994 – are in China. Another 78 cases are confirmed outside of China in 16 countries. Countries with the most confirmed cases include Thailand with 14 and Taiwan and South Korea with 8 each. All reported deaths have been in China, and include 16 healthcare workers.\n\nWhen did the outbreak start?\nChina first reported the outbreak in Wuhan on Dec. 30, 2019.\n\nWhat are public officials doing to contain the virus?\nOn Tuesday, the CDC urged travelers to avoid non-essential travel to China.\n\nChinese officials have shut down all public transportation to 10 cities, affecting 35 million people. The first was Wuhan, which has a population of about 11 million. In Wuhan, that includes buses, subways, trains, and the airport.\n\nAll passengers flying into the United States from Wuhan will be routed through one of five airports and screened for fever and other signs of the virus. “Patients of concern��� will be referred to a facility and given a test to find out if they have the virus.\n\nWhat do we know about cases in the United States?\nIn all U.S. cases so far, patients had recently traveled to Wuhan.\n\nCalifornia has two patients, one in Los Angeles County and one in Orange County. The patient from Orange County is a man in his 50s. He is in a local hospital in isolation and is in good condition, according to the Orange County Health Care Agency. Los Angeles County officials did not provide additional details about the patient there.\n\nArizona\'s Department of Health Services said its patient is a Maricopa County resident and member of the Arizona State University community who did not live in student housing. The patient is not severely ill and is being kept in isolation. \n\nAnother case involves a woman in her 60s from Chicago. The Chicago Department of Public Health reported that she had visited Wuhan, China in December and returned to Chicago earlier this month. She is hospitalized in stable condition. \n\nThe first U.S. patient is a man in his 30s from Washington state. He had traveled from Wuhan and entered the country before the screening was in place. He started having symptoms and contacted his doctor. He is in good condition and is in isolation at Providence Regional Medical Center.\n\nIs travel to China safe?\nWuhan is closed to travelers.\n\nThe CDC is advising that travelers avoid non-essential travel to China. Travelers who do go should:\n\nAvoid contact with sick people.\nAvoid animals, animal markets, and products that come from animals.\nWash their hands often with soap and water, or use an alcohol-based sanitizer if that’s not available.\nSeek medical care right away for a fever, cough, or difficulty breathing. Tell a health care professional about any travel.\nWhat are the symptoms, and how is the virus diagnosed?\nChina created a test for the virus and shared that information with other countries. The CDC has developed its own test.\n\nSymptoms include a fever, coughing, and shortness of breath. They may appear 2 to 14 days after you’re exposed to the virus.\n\nWhat is the source of the virus, and how is it spread?\nHealth officials are not sure of the source of the virus yet or how easily it can spread. Coronaviruses are found in many different animals, including camels, cattle, cats, and bats. One research paper also suggested snakes as a possible source. The new virus may be linked to a seafood and live animal market in Wuhan that has since been closed\n\nThe virus can spread from person to person. Health officials are seeing this happen most often where people are close together and in health care settings. To date, 16 health care workers have been infected.\n\nThe CDC believes that severe acute respiratory syndrome (SARS) and Middle East respiratory syndrome (MERS), two other types of coronavirus, are spread through droplets when someone coughs or sneezes.\n\nIs there a vaccine?\nThere is no vaccine, but the National Institutes of Health is working on one and hopes to begin testing in several months. That testing would be for safety. If it’s safe, there would be testing to see how well it works.\n\nHow is it treated?\nThere is no specific treatment for the virus. Patients are generally given supportive care for their symptoms, such a fluids and pain relievers. Hospitalized patients may need support with breathing.'

Process data and calculate indices

```
chars = list(set(data))
data_size, X_size = len(data), len(chars)
print("Corona Virus article has %d characters, %d unique characters" %(data_size, X_size))
char_to_idx = {ch:i for i,ch in enumerate(chars)}
idx_to_char = {i:ch for i,ch in enumerate(chars)}
```

    Corona Virus article has 10223 characters, 75 unique characters

# Constants and Hyperparameters

```
Hidden_Layer_size = 10 #size of the hidden layer
Time_steps = 10 # Number of time steps (length of the sequence) used for training
learning_rate = 1e-1 # Learning Rate
weight_sd = 0.1 #Standard deviation of weights for initialization
z_size = Hidden_Layer_size + X_size #Size of concatenation(H, X) vector
```

# Activation Functions and Derivatives

```
def sigmoid(x): # sigmoid function
    z = 1 / (1 + np.exp(-x))
    return z

def dsigmoid(y): # derivative of sigmoid function
    dz = y * (1 - y)
    return dz

def tanh(x): # tanh function
    z = np.tanh(x)
    return z

def dtanh(y): # derivative of tanh
    dz = 1 - y * y
    return dz
```

#### Sigmoid

\begin{align}
\sigma(x) &= \frac{1}{1 + e^{-x}}\\
\frac{d\sigma(x)}{dx} &= \sigma(x) \cdot (1 - \sigma(x))
\end{align}

#### Tanh

\begin{align}
\frac{d\text{tanh}(x)}{dx} &= 1 - \text{tanh}^2(x)
\end{align}

# Quiz Question 1

What is the value of sigmoid(0) calculated from  your code? (Answer up to 1 decimal point, e.g. 4.2 and NOT 4.29999999, no rounding off).

```
sigmoid(0)
```

    0.5

# Quiz Question 2

What is the value of dsigmoid(sigmoid(0)) calculated from your code?? (Answer up to 2 decimal point, e.g. 4.29 and NOT 4.29999999, no rounding off).

```
dsigmoid(sigmoid(0))
```

    0.25

# Quiz Question 3

What is the value of tanh(dsigmoid(sigmoid(0))) calculated from your code?? (Answer up to 5 decimal point, e.g. 4.29999 and NOT 4.29999999, no rounding off).

```
tanh(dsigmoid(sigmoid(0)))
```

    0.24491866240370913

# Quiz Question 4

What is the value of dtanh(tanh(dsigmoid(sigmoid(0)))) calculated from your code?? (Answer up to 5 decimal point, e.g. 4.29999 and NOT 4.29999999, no rounding off).

```
dtanh(tanh(dsigmoid(sigmoid(0))))
```

    0.940014848806378

# Parameters

```
class Param:
    def __init__(self, name, value):
      self.name = name
      self.v = value # parameter value
      self.d = np.zeros_like(value) # derivative
      self.m = np.zeros_like(value) # momentum for Adagrad
```

We use random weights with normal distribution (0, weight_sd) for  tanh  activation function and (0.5, weight_sd) for  `sigmoid`  activation function.

Biases are initialized to zeros.

# LSTM
You are making this network, please note f, i, c and o (also "v") in the image below:

Please note that we are concatenating the old_hidden_vector and new_input.

# Quiz Question 4

In the class definition below, what should be size_a, size_b, and size_c? ONLY use the variables defined above.

```
size_a = Hidden_Layer_size
size_b = z_size
size_c = X_size

class Parameters:
    def __init__(self):
        self.W_f = Param('W_f', np.random.randn(size_a, size_b) * weight_sd + 0.5)
        self.b_f = Param('b_f', np.zeros((size_a, 1)))

        self.W_i = Param('W_i', np.random.randn(size_a, size_b) * weight_sd + 0.5)
        self.b_i = Param('b_i', np.zeros((size_a, 1)))

        self.W_C = Param('W_C', np.random.randn(size_a, size_b) * weight_sd)
        self.b_C = Param('b_C', np.zeros((size_a, 1)))

        self.W_o = Param('W_o', np.random.randn(size_a, size_b) * weight_sd + 0.5)
        self.b_o = Param('b_o', np.zeros((size_a, 1)))

        #For final layer to predict the next character
        self.W_v = Param('W_v', np.random.randn(X_size, size_a) * weight_sd)
        self.b_v = Param('b_v', np.zeros((size_c, 1)))

    def all(self):
        return [self.W_f, self.W_i, self.W_C, self.W_o, self.W_v,
               self.b_f, self.b_i, self.b_C, self.b_o, self.b_v]

parameters = Parameters()
```

Look at these operations which we'll be writing:

**Concatenation of h and x:**

$z\:=\:\left[h_{t-1},\:x\right]$

$f_t=\sigma\left(W_f\cdot z\:+\:b_f\:\right)$

$i_i=\sigma\left(W_i\cdot z\:+\:b_i\right)$

$\overline{C_t}=\tanh\left(W_C\cdot z\:+\:b_C\right)$

$C_t=f_t\ast C_{t-1}+i_t\ast \overline{C}_t$

$o_t=\sigma\left(W_o\cdot z\:+\:b_i\right)$

$h_t=o_t\ast\tanh\left(C_t\right)$

**Logits:**

$v_t=W_v\cdot h_t+b_v$

**Softmax:**

$\hat{y}=softmax\left(v_t\right)$

```
def forward(x, h_prev, C_prev, p = parameters):
    assert x.shape == (X_size, 1)
    assert h_prev.shape == (Hidden_Layer_size, 1)
    assert C_prev.shape == (Hidden_Layer_size, 1)

    z = np.row_stack((h_prev, x))
    f = sigmoid(np.dot(p.W_f.v, z) + p.b_f.v)
    i = sigmoid(np.dot(p.W_i.v, z) + p.b_i.v)
    C_bar = tanh(np.dot(p.W_C.v, z) + p.b_C.v)

    C = f * C_prev + i * C_bar
    o = sigmoid(np.dot(p.W_o.v, z) + p.b_o.v)
    h = o * tanh(C)

    v = np.dot(p.W_v.v, h) + p.b_v.v
    y = np.exp(v) / np.sum(np.exp(v)) # softmax

    return z, f, i, C_bar, C, o, h, v, y
```

You must finish the function above before you can attempt the questions below.

# Quiz Question 5

What is the output of 'print(len(forward(np.zeros((X_size, 1)), np.zeros((Hidden_Layer_size, 1)), np.zeros((Hidden_Layer_size, 1)), parameters)))'?

```
print(len(forward(np.zeros((X_size, 1)), np.zeros((Hidden_Layer_size, 1)), np.zeros((Hidden_Layer_size, 1)), parameters)))
```

    9

# Quiz Question 6.

Assuming you have fixed the forward function, run this command:
z, f, i, C_bar, C, o, h, v, y = forward(np.zeros((X_size, 1)), np.zeros((Hidden_Layer_size, 1)), np.zeros((Hidden_Layer_size, 1)))

Now, find these values:

1.   print(z.shape)
2.   print(np.sum(z))
3.   print(np.sum(f))

Copy and paste exact values you get in the logs into the quiz.

```
z, f, i, C_bar, C, o, h, v, y = forward(np.zeros((X_size, 1)), np.zeros((Hidden_Layer_size, 1)), np.zeros((Hidden_Layer_size, 1)))
```

```
z.shape, np.sum(z), np.sum(f)
```

    ((85, 1), 0.0, 5.0)

# Backpropagation

Here we are defining the backpropagation. It's too complicated, here is the whole code. (Please note that this would work only if your earlier code is perfect).

```
def backward(target, dh_next, dC_next, C_prev,
             z, f, i, C_bar, C, o, h, v, y,
             p = parameters):

    assert z.shape == (X_size + Hidden_Layer_size, 1)
    assert v.shape == (X_size, 1)
    assert y.shape == (X_size, 1)

    for param in [dh_next, dC_next, C_prev, f, i, C_bar, C, o, h]:
        assert param.shape == (Hidden_Layer_size, 1)

    dv = np.copy(y)
    dv[target] -= 1

    p.W_v.d += np.dot(dv, h.T)
    p.b_v.d += dv

    dh = np.dot(p.W_v.v.T, dv)
    dh += dh_next
    do = dh * tanh(C)
    do = dsigmoid(o) * do
    p.W_o.d += np.dot(do, z.T)
    p.b_o.d += do

    dC = np.copy(dC_next)
    dC += dh * o * dtanh(tanh(C))
    dC_bar = dC * i
    dC_bar = dtanh(C_bar) * dC_bar
    p.W_C.d += np.dot(dC_bar, z.T)
    p.b_C.d += dC_bar

    di = dC * C_bar
    di = dsigmoid(i) * di
    p.W_i.d += np.dot(di, z.T)
    p.b_i.d += di

    df = dC * C_prev
    df = dsigmoid(f) * df
    p.W_f.d += np.dot(df, z.T)
    p.b_f.d += df

    dz = (np.dot(p.W_f.v.T, df)
         + np.dot(p.W_i.v.T, di)
         + np.dot(p.W_C.v.T, dC_bar)
         + np.dot(p.W_o.v.T, do))
    dh_prev = dz[:Hidden_Layer_size, :]
    dC_prev = f * dC

    return dh_prev, dC_prev
```

# Forward and Backward Combined Pass

Let's first clear the gradients before each backward pass

```
def clear_gradients(params = parameters):
    for p in params.all():
        p.d.fill(0)
```

Clip gradients to mitigate exploding gradients

```
def clip_gradients(params = parameters):
    for p in params.all():
        np.clip(p.d, -1, 1, out=p.d)
```

Calculate and store the values in forward pass. Accumulate gradients in backward pass and clip gradients to avoid exploding gradients.

input, target are list of integers, with character indexes.
h_prev is the array of initial h at  h−1  (size H x 1)
C_prev is the array of initial C at  C−1  (size H x 1)
Returns loss, final  hT  and  CT

```
def forward_backward(inputs, targets, h_prev, C_prev):
    global paramters

    # To store the values for each time step
    x_s, z_s, f_s, i_s,  = {}, {}, {}, {}
    C_bar_s, C_s, o_s, h_s = {}, {}, {}, {}
    v_s, y_s =  {}, {}

    # Values at t - 1
    h_s[-1] = np.copy(h_prev)
    C_s[-1] = np.copy(C_prev)

    loss = 0
    # Loop through time steps
    assert len(inputs) == Time_steps
    for t in range(len(inputs)):
        x_s[t] = np.zeros((X_size, 1))
        x_s[t][inputs[t]] = 1 # Input character

        (z_s[t], f_s[t], i_s[t],
        C_bar_s[t], C_s[t], o_s[t], h_s[t],
        v_s[t], y_s[t]) = \
            forward(x_s[t], h_s[t - 1], C_s[t - 1]) # Forward pass

        loss += -np.log(y_s[t][targets[t], 0]) # Loss for at t

    clear_gradients()

    dh_next = np.zeros_like(h_s[0]) #dh from the next character
    dC_next = np.zeros_like(C_s[0]) #dh from the next character

    for t in reversed(range(len(inputs))):
        # Backward pass
        dh_next, dC_next = \
            backward(target = targets[t], dh_next = dh_next,
                     dC_next = dC_next, C_prev = C_s[t-1],
                     z = z_s[t], f = f_s[t], i = i_s[t], C_bar = C_bar_s[t],
                     C = C_s[t], o = o_s[t], h = h_s[t], v = v_s[t],
                     y = y_s[t])

    clip_gradients()

    return loss, h_s[len(inputs) - 1], C_s[len(inputs) - 1]
```

# Sample the next character

```
def sample(h_prev, C_prev, first_char_idx, sentence_length):
    x = np.zeros((X_size, 1))
    x[first_char_idx] = 1

    h = h_prev
    C = C_prev

    indexes = []

    for t in range(sentence_length):
        _, _, _, _, C, _, h, _, p = forward(x, h, C)
        idx = np.random.choice(range(X_size), p=p.ravel())
        x = np.zeros((X_size, 1))
        x[idx] = 1
        indexes.append(idx)

    return indexes
```

# Training (Adagrad)

Update the graph and display a sample output

```
def update_status(inputs, h_prev, C_prev):
    #initialized later
    global plot_iter, plot_loss
    global smooth_loss

    # Get predictions for 200 letters with current model

    sample_idx = sample(h_prev, C_prev, inputs[0], 200)
    txt = ''.join(idx_to_char[idx] for idx in sample_idx)

    # Clear and plot
    plt.plot(plot_iter, plot_loss)
    display.clear_output(wait=True)
    plt.show()

    #Print prediction and loss
    print("----\n %s \n----" % (txt, ))
    print("iter %d, loss %f" % (iteration, smooth_loss))
```

# Update Parameters

\begin{align}
\theta_i &= \theta_i - \eta\frac{d\theta_i}{\sum dw_{\tau}^2} \\
d\theta_i &= \frac{\partial L}{\partial \theta_i}
\end{align}

```
def update_paramters(params = parameters):
    for p in params.all():
        p.m += p.d * p.d # Calculate sum of gradients
        #print(learning_rate * dparam)
        p.v += -(learning_rate * p.d / np.sqrt(p.m + 1e-8))
```

To delay the keyboard interrupt to prevent the training from stopping in the middle of an iteration

```
# Exponential average of loss
# Initialize to a error of a random model
smooth_loss = -np.log(1.0 / X_size) * Time_steps

iteration, pointer = 0, 0

# For the graph
plot_iter = np.zeros((0))
plot_loss = np.zeros((0))
```

# Training Loop

```
iter = 1000
while iter > 0:
  # Reset
  if pointer + Time_steps >= len(data) or iteration == 0:
      g_h_prev = np.zeros((Hidden_Layer_size, 1))
      g_C_prev = np.zeros((Hidden_Layer_size, 1))
      pointer = 0

  inputs = ([char_to_idx[ch]
              for ch in data[pointer: pointer + Time_steps]])
  targets = ([char_to_idx[ch]
              for ch in data[pointer + 1: pointer + Time_steps + 1]])

  loss, g_h_prev, g_C_prev = \
      forward_backward(inputs, targets, g_h_prev, g_C_prev)
  smooth_loss = smooth_loss * 0.999 + loss * 0.001

  # Print every hundred steps
  if iteration % 100 == 0:
      update_status(inputs, g_h_prev, g_C_prev)

  update_paramters()

  plot_iter = np.append(plot_iter, [iteration])
  plot_loss = np.append(plot_loss, [loss])

  pointer += Time_steps
  iteration += 1
  iter = iter -1
```

# Quiz Question 7.

Run the above code for 50000 iterations making sure that you have 100 hidden layers and time_steps is 40. What is the loss value you're seeing?

```
Hidden_Layer_size = 100 #size of the hidden layer
Time_steps = 40 # Number of time steps (length of the sequence) used for training
learning_rate = 1e-1 # Learning Rate
weight_sd = 0.1 #Standard deviation of weights for initialization
z_size = Hidden_Layer_size + X_size #Size of concatenation(H, X) vector
```

```
class Param:
    def __init__(self, name, value):
      self.name = name
      self.v = value # parameter value
      self.d = np.zeros_like(value) # derivative
      self.m = np.zeros_like(value) # momentum for Adagrad
```

```
size_a = Hidden_Layer_size
size_b = z_size
size_c = X_size

class Parameters:
    def __init__(self):
        self.W_f = Param('W_f', np.random.randn(size_a, size_b) * weight_sd + 0.5)
        self.b_f = Param('b_f', np.zeros((size_a, 1)))

        self.W_i = Param('W_i', np.random.randn(size_a, size_b) * weight_sd + 0.5)
        self.b_i = Param('b_i', np.zeros((size_a, 1)))

        self.W_C = Param('W_C', np.random.randn(size_a, size_b) * weight_sd)
        self.b_C = Param('b_C', np.zeros((size_a, 1)))

        self.W_o = Param('W_o', np.random.randn(size_a, size_b) * weight_sd + 0.5)
        self.b_o = Param('b_o', np.zeros((size_a, 1)))

        #For final layer to predict the next character
        self.W_v = Param('W_v', np.random.randn(X_size, size_a) * weight_sd)
        self.b_v = Param('b_v', np.zeros((size_c, 1)))

    def all(self):
        return [self.W_f, self.W_i, self.W_C, self.W_o, self.W_v,
               self.b_f, self.b_i, self.b_C, self.b_o, self.b_v]

parameters = Parameters()
```

```
def forward(x, h_prev, C_prev, p = parameters):
    assert x.shape == (X_size, 1)
    assert h_prev.shape == (Hidden_Layer_size, 1)
    assert C_prev.shape == (Hidden_Layer_size, 1)

    z = np.row_stack((h_prev, x))
    f = sigmoid(np.dot(p.W_f.v, z) + p.b_f.v)
    i = sigmoid(np.dot(p.W_i.v, z) + p.b_i.v)
    C_bar = tanh(np.dot(p.W_C.v, z) + p.b_C.v)

    C = f * C_prev + i * C_bar
    o = sigmoid(np.dot(p.W_o.v, z) + p.b_o.v)
    h = o * tanh(C)

    v = np.dot(p.W_v.v, h) + p.b_v.v
    y = np.exp(v) / np.sum(np.exp(v)) # softmax

    return z, f, i, C_bar, C, o, h, v, y
```

```
z, f, i, C_bar, C, o, h, v, y = forward(np.zeros((X_size, 1)), np.zeros((Hidden_Layer_size, 1)), np.zeros((Hidden_Layer_size, 1)))
```

```
z.shape, np.sum(z), np.sum(f)
```

    ((175, 1), 0.0, 50.0)

```
def backward(target, dh_next, dC_next, C_prev,
             z, f, i, C_bar, C, o, h, v, y,
             p = parameters):

    assert z.shape == (X_size + Hidden_Layer_size, 1)
    assert v.shape == (X_size, 1)
    assert y.shape == (X_size, 1)

    for param in [dh_next, dC_next, C_prev, f, i, C_bar, C, o, h]:
        assert param.shape == (Hidden_Layer_size, 1)

    dv = np.copy(y)
    dv[target] -= 1

    p.W_v.d += np.dot(dv, h.T)
    p.b_v.d += dv

    dh = np.dot(p.W_v.v.T, dv)
    dh += dh_next
    do = dh * tanh(C)
    do = dsigmoid(o) * do
    p.W_o.d += np.dot(do, z.T)
    p.b_o.d += do

    dC = np.copy(dC_next)
    dC += dh * o * dtanh(tanh(C))
    dC_bar = dC * i
    dC_bar = dtanh(C_bar) * dC_bar
    p.W_C.d += np.dot(dC_bar, z.T)
    p.b_C.d += dC_bar

    di = dC * C_bar
    di = dsigmoid(i) * di
    p.W_i.d += np.dot(di, z.T)
    p.b_i.d += di

    df = dC * C_prev
    df = dsigmoid(f) * df
    p.W_f.d += np.dot(df, z.T)
    p.b_f.d += df

    dz = (np.dot(p.W_f.v.T, df)
         + np.dot(p.W_i.v.T, di)
         + np.dot(p.W_C.v.T, dC_bar)
         + np.dot(p.W_o.v.T, do))
    dh_prev = dz[:Hidden_Layer_size, :]
    dC_prev = f * dC

    return dh_prev, dC_prev
```

```
def clear_gradients(params = parameters):
    for p in params.all():
        p.d.fill(0)
```

```
def clip_gradients(params = parameters):
    for p in params.all():
        np.clip(p.d, -1, 1, out=p.d)
```

```
def forward_backward(inputs, targets, h_prev, C_prev):
    global paramters

    # To store the values for each time step
    x_s, z_s, f_s, i_s,  = {}, {}, {}, {}
    C_bar_s, C_s, o_s, h_s = {}, {}, {}, {}
    v_s, y_s =  {}, {}

    # Values at t - 1
    h_s[-1] = np.copy(h_prev)
    C_s[-1] = np.copy(C_prev)

    loss = 0
    # Loop through time steps
    assert len(inputs) == Time_steps
    for t in range(len(inputs)):
        x_s[t] = np.zeros((X_size, 1))
        x_s[t][inputs[t]] = 1 # Input character

        (z_s[t], f_s[t], i_s[t],
        C_bar_s[t], C_s[t], o_s[t], h_s[t],
        v_s[t], y_s[t]) = \
            forward(x_s[t], h_s[t - 1], C_s[t - 1]) # Forward pass

        loss += -np.log(y_s[t][targets[t], 0]) # Loss for at t

    clear_gradients()

    dh_next = np.zeros_like(h_s[0]) #dh from the next character
    dC_next = np.zeros_like(C_s[0]) #dh from the next character

    for t in reversed(range(len(inputs))):
        # Backward pass
        dh_next, dC_next = \
            backward(target = targets[t], dh_next = dh_next,
                     dC_next = dC_next, C_prev = C_s[t-1],
                     z = z_s[t], f = f_s[t], i = i_s[t], C_bar = C_bar_s[t],
                     C = C_s[t], o = o_s[t], h = h_s[t], v = v_s[t],
                     y = y_s[t])

    clip_gradients()

    return loss, h_s[len(inputs) - 1], C_s[len(inputs) - 1]
```

```
def sample(h_prev, C_prev, first_char_idx, sentence_length):
    x = np.zeros((X_size, 1))
    x[first_char_idx] = 1

    h = h_prev
    C = C_prev

    indexes = []

    for t in range(sentence_length):
        _, _, _, _, C, _, h, _, p = forward(x, h, C)
        idx = np.random.choice(range(X_size), p=p.ravel())
        x = np.zeros((X_size, 1))
        x[idx] = 1
        indexes.append(idx)

    return indexes
```

```
def update_status(inputs, h_prev, C_prev):
    #initialized later
    global plot_iter, plot_loss
    global smooth_loss

    # Get predictions for 200 letters with current model

    sample_idx = sample(h_prev, C_prev, inputs[0], 200)
    txt = ''.join(idx_to_char[idx] for idx in sample_idx)

    # Clear and plot
    plt.plot(plot_iter, plot_loss)
    display.clear_output(wait=True)
    plt.show()

    #Print prediction and loss
    print("----\n %s \n----" % (txt, ))
    print("iter %d, loss %f" % (iteration, smooth_loss))
```

```
def update_paramters(params = parameters):
    for p in params.all():
        p.m += p.d * p.d # Calculate sum of gradients
        #print(learning_rate * dparam)
        p.v += -(learning_rate * p.d / np.sqrt(p.m + 1e-8))
```

```
# Exponential average of loss
# Initialize to a error of a random model
smooth_loss = -np.log(1.0 / X_size) * Time_steps

iteration, pointer = 0, 0

# For the graph
plot_iter = np.zeros((0))
plot_loss = np.zeros((0))
```

```
iter = 50000
while iter > 0:
  # Reset
  if pointer + Time_steps >= len(data) or iteration == 0:
      g_h_prev = np.zeros((Hidden_Layer_size, 1))
      g_C_prev = np.zeros((Hidden_Layer_size, 1))
      pointer = 0

  inputs = ([char_to_idx[ch]
              for ch in data[pointer: pointer + Time_steps]])
  targets = ([char_to_idx[ch]
              for ch in data[pointer + 1: pointer + Time_steps + 1]])

  loss, g_h_prev, g_C_prev = \
      forward_backward(inputs, targets, g_h_prev, g_C_prev)
  smooth_loss = smooth_loss * 0.999 + loss * 0.001

  # Print every hundred steps
  if iteration % 100 == 0:
      update_status(inputs, g_h_prev, g_C_prev)

  update_paramters()

  plot_iter = np.append(plot_iter, [iteration])
  plot_loss = np.append(plot_loss, [loss])

  pointer += Time_steps
  iteration += 1
  iter = iter -1
```

```

```