# LASSO Regresion on covid-19 cases in New York, Massachusetts and Connecticut

## Load modulues

```python
%matplotlib inline
import os
import requests
import urllib.parse
from zipfile import ZipFile
import io
from datetime import datetime, timedelta

import pandas as pd
import numpy as np
import statsmodels.api as sm
import scipy

import seaborn as sns
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
from statsmodels.sandbox.regression.predstd import wls_prediction_std
from sklearn import linear_model
from scipy import stats as sps
from sklearn.feature_selection import SelectFromModel
from IPython.display import display

locator = mdates.AutoDateLocator()

formatter = mdates.ConciseDateFormatter(locator)

sns.set_style("whitegrid")
GAMMA = 1/7.5

events = {}

df_dict = {}

if not os.path.exists('data'):
    os.mkdir('data')
if not os.path.exists('figs'):
    os.mkdir('figs')
```

    /home/andres/anaconda3/lib/python3.7/site-packages/statsmodels/tools/_testing.py:19: FutureWarning: pandas.util.testing is deprecated. Use the functions in the public API at pandas.testing instead.
      import pandas.util.testing as tm

## Important dates

## Get data

### New York

```python
dsname = 'New York'

# API information
# https://health.data.ny.gov/Health/New-York-State-Statewide-COVID-19-Testing/xdss-u53e

if not os.path.isfile('data/ny.csv'):
    url = urllib.parse.quote("https://health.data.ny.gov/resource/xdss-u53e.csv/?$limit=5000")
    #
    dfs = []
    offset = 0
    while offset >=0:
        url = "https://health.data.ny.gov/resource/xdss-u53e.csv/?$limit=5000&$offset={}".format(offset)
        df = pd.read_csv(url, usecols=['test_date', 'total_number_of_tests', 'new_positives'])
        dfs.append(df)
        if len(df)==5000:
            offset += 5000
        else:
            offset = -1
    dfraw = pd.concat(dfs)
    #'test_date=2020-03-15T00:00:00.000'

    dfraw = dfraw.rename(columns={'new_positives': 'Positives', 'total_number_of_tests': 'Tests', 'test_date': 'date'})
    print(len(dfraw))
    dfraw['date'] = pd.to_datetime(dfraw['date'])
    #counties = (df.groupby('date')['Tests']>0).count()
    df = dfraw.groupby('date').sum()
    print(df.head(1))
    df['Odds'] = df.Positives / (df.Tests - df.Positives)
    df['Date'] = pd.to_datetime(df.index)
    df.to_csv('data/ny.csv')
    df = df[df['Date'] >= '2020-03-15']
    # last date of full NYS PAUSE
    df = df[df['Date'] <= '2020-05-15']

else:
    df = pd.read_csv('data/ny.csv')
    df['date'] =pd.to_datetime(df['date'])
    df['Date'] = pd.to_datetime(df['Date'])
    df = df.set_index('date')
    df = df[df['Date'] >= '2020-03-15']
    # last date of full NYS PAUSE
    df = df[df['Date'] <= '2020-05-15']
df = df[['Date', 'Tests', 'Positives', 'Odds']]
df_dict[dsname] = df

# 03/17/2020 close of gyms, restaurants and bars, movie theaters, mass gathering up to 50. https://www.governor.ny.gov/news/amid-lack-federal-direction-governor-cuomo-governor-murphy-and-governor-lamont-announce
bars = pd.to_datetime('03-16-2020 20:00', dayfirst=False)
# 03/18/2020 school clousure http://www.nysed.gov/news/2020/state-education-department-issues-updated-guidance-schools-regarding-novel-coronavirus
schools = pd.to_datetime('03-18-2020 20:00', dayfirst=False)

# https://www.governor.ny.gov/news/amid-ongoing-covid-19-pandemic-governor-cuomo-announces-deployment-1000-bed-hospital-ship-usns
# 03/20/2020 00:00 50% of the workforce
workforce_50 = pd.to_datetime('03-20-2020 20:00', dayfirst=False)
# 03/22/2020 20:00 ny_pause
ny_pause = pd.to_datetime('22-03-2020 00:00', dayfirst=True)
# CDC masks https://www.npr.org/sections/goatsandsoda/2020/04/10/829890635/why-there-so-many-different-guidelines-for-face-masks-for-the-public
masks_cdc = pd.to_datetime('03-04-2020 00:00', dayfirst=True)
mask_employers = pd.to_datetime('12-04-2020 00:00', dayfirst=True)
mask_public = pd.to_datetime('17-04-2020 00:00', dayfirst=True)

events_list = [bars, schools, workforce_50, ny_pause, masks_cdc, mask_employers, mask_public]
events['New York'] = events_list

df.tail()
```

      Date
      Tests
      Positives
      Odds

      date

      2020-05-11
      2020-05-11
      20463
      1430
      0.075133

      2020-05-12
      2020-05-12
      33794
      2176
      0.068822

      2020-05-13
      2020-05-13
      39850
      2390
      0.063801

      2020-05-14
      2020-05-14
      39291
      2762
      0.075611

      2020-05-15
      2020-05-15
      40669
      2419
      0.063242

## Connecticut

```python
dsname = 'Connecticut'
# reopening phase 1 may 20
# https://portal.ct.gov/Coronavirus/Covid-19-Knowledge-Base/Reopen-plan

# all APIs avaliables
# https://data.ct.gov/stories/s/COVID-19-data/wa3g-tfvc/#covid-19-testing-data

if not os.path.isfile('data/connecticut.csv'):
    dfs = []
    offset = 0
    while offset >=0:
        url = 'https://data.ct.gov/resource/qfkt-uahj.csv?$limit=5000&$offset={}'.format(offset)
        df = pd.read_csv(url)
        dfs.append(df)
        if len(df)==5000:
            offset += 5000
        else:
            offset = -1
    dfcounty = pd.concat(dfs)
    dfcounty = dfcounty.rename(columns={'number_of_positives': 'Positives', 'number_of_tests': 'Tests', 'number_of_negatives': 'Negatives'})
    dfcounty['Tests'] = dfcounty['Tests'] - dfcounty['number_of_indeterminates']
    # print(len(dfcounty))
    dfcounty['date'] = pd.to_datetime(dfcounty['date'])
    df = dfcounty.groupby('date').sum()
    #df = df.diff()
    df['Odds'] = df.Positives / df.Negatives
    df['Date'] = pd.to_datetime(df.index)
    df.to_csv('data/connecticut.csv')
else:
    df = pd.read_csv('data/connecticut.csv')
    df = df.set_index('date')
df['Date'] = pd.to_datetime(df['Date'])
df.index = pd.to_datetime(df.index)
df = df.sort_index()
#print(df[['Tests', 'Positives', 'Odds']].head(30))

df = df[df['Date'] <= '2020-05-19']
df = df[df['Date'] > '2020-03-19']
df = df[['Date', 'Tests', 'Positives', 'Odds']]
#print(df.head())
df_dict[dsname] = df
# intervention dates
# https://portal.ct.gov/Coronavirus/Pages/Emergency-Orders-issued-by-the-Governor-and-State-Agencies
# 03/12/2020 no gatherings with more than 250
large_gatherings = pd.to_datetime('03-12-2020 00:00', dayfirst=False)
schools = pd.to_datetime('03-17-2020 00:00', dayfirst=False)
bars = pd.to_datetime('03-16-2020 20:00', dayfirst=False)

malls = pd.to_datetime('03-19-2020 20:00', dayfirst=False)
workforce100 = pd.to_datetime('03-23-2020 20:00', dayfirst=False)
# CDC masks https://www.npr.org/sections/goatsandsoda/2020/04/10/829890635/why-there-so-many-different-guidelines-for-face-masks-for-the-public
masks_cdc = pd.to_datetime('04-03-2020 00:00', dayfirst=False)
mask_public = pd.to_datetime('04-20-2020 00:00', dayfirst=False)

event_list = [large_gatherings, bars, schools, malls, workforce100, masks_cdc,  mask_public]
events[dsname] = event_list

df.tail()

```

      Date
      Tests
      Positives
      Odds

      date

      2020-05-15
      2020-05-15
      7492
      650
      0.095001

      2020-05-16
      2020-05-16
      3756
      315
      0.091543

      2020-05-17
      2020-05-17
      2607
      210
      0.087610

      2020-05-18
      2020-05-18
      6759
      703
      0.116083

      2020-05-19
      2020-05-19
      6971
      793
      0.128359

## Massachusetts

```python
dsname = 'Massachusetts'
# web site with data
# https://www.mass.gov/info-details/covid-19-response-reporting
yesterday = datetime.today() - timedelta(0)
if  not os.path.isfile('data/massachusetts.csv'):
    #yesterday = datetime.today() - timedelta(10)
    yesterday_str = datetime.strftime(yesterday, '%B-%d-%Y').lower()
    fn = 'data/{}.zip'.format(dsname)

    url2 = 'https://www.mass.gov/doc/covid-19-raw-data-june-10-2020/download'
    url = 'https://www.mass.gov/doc/covid-19-raw-data-{}/download'.format(yesterday_str)
    print(url)
    print(url2)
    myfile = requests.get(url, allow_redirects=True)
    #open(fn, 'wb').write(myfile.content)
    zf = ZipFile(io.BytesIO(myfile.content))
    csvf = zf.open('TestingByDate.csv')
    df = pd.read_csv(csvf)

    # https://www.mass.gov/doc/covid-19-raw-data-may-27-2020/download
    #df = pd.read_csv('data/massachusetts/COVID-19-Dashboard-Files-05-24-2020/TestingByDate.csv',
    #                 usecols=['Date', 'New', 'Positive'])

    df = df.rename(columns={'Molecular Positive New': 'Positives', 'Molecular New': 'Tests'})
    df['Negatives'] = df.Tests - df.Positives
    df = df.query('Tests>100')
    df['date'] = pd.to_datetime(df['Date'])
    df= df.set_index('date')
    df['Date'] = pd.to_datetime(df['Date'])
    df['Odds'] = df.Positives / df.Negatives
    #df = df[df['Date']<'2020-05-15']
    df.to_csv('data/massachusetts.csv')
else:
    df = pd.read_csv('data/massachusetts.csv')
    df['Date'] = pd.to_datetime(df['Date'])
    df['date'] = pd.to_datetime(df['date'])
    df = df.set_index('date')
df = df[df['Date']<'2020-05-18']
#df = df[df['Date']>'2020-03-15']
df = df[df['Date']>'2020-03-14']
df = df[['Date', 'Tests', 'Positives', 'Odds']]
df_dict[dsname] = df

# https://www.mass.gov/info-details/covid-19-state-of-emergency
schools = pd.to_datetime('03-22-2020 00:00', dayfirst=False)
saty_at_home = pd.to_datetime('03-24-2020 00:00', dayfirst=False)
masks_cdc = pd.to_datetime('04-03-2020 00:00', dayfirst=False)
masks_public = pd.to_datetime('05-06-2020 00:00', dayfirst=False)

event_list = [schools, saty_at_home, masks_cdc, masks_public]
events[dsname] = event_list
df_dict[dsname] = df
```

## Michigan

```python
dsname = 'Michigan'
if not os.path.isfile('data/michigan.csv'):
    df = pd.read_excel('https://www.michigan.gov/documents/coronavirus/Diagnostic_Tests_by_Result_and_County_2020-06-16_693915_7.xlsx')
    df['date'] = pd.to_datetime(df['MessageDate'])
    df = df[df['COUNTY']!='Correctional']
    df = df.groupby('date').sum()
    df['Date'] = df.index
    df.to_csv('data/michigan.csv')
else:
    df = pd.read_csv('data/michigan.csv', index_col='date', parse_dates=['date', 'Date'])
df = df.rename(columns={'Negative': 'Negatives', 'Positive': 'Positives','Total': 'Tests'})
df['Odds'] = df.Positives / df.Negatives
df = df[df['Date']>'2020-03-16']
df = df[df['Date']<'2020-05-18']
df_dict[dsname] = df

# https://www.michigan.gov/whitmer/0,9309,7-387-90499_90705-522626--,00.html
saty_at_home = pd.to_datetime('03-24-2020 00:00', dayfirst=False)
masks_cdc = pd.to_datetime('04-03-2020 00:00', dayfirst=False)
# https://www.michigan.gov/whitmer/0,9309,7-387-90499-526896--,00.html
masks_public = pd.to_datetime('04-24-2020 00:00', dayfirst=False)
event_list = [saty_at_home, masks_cdc, masks_public]
events[dsname] = event_list
```

## Rhode Island

```python
dsname = 'Rhode Island'
#https://docs.google.com/spreadsheets/d/1n-zMS9Al94CPj_Tc3K7Adin-tN9x1RSjjx2UzJ4SV7Q/edit#gid=590763272
url = 'https://docs.google.com/spreadsheets/d/1n-zMS9Al94CPj_Tc3K7Adin-tN9x1RSjjx2UzJ4SV7Q/export?format=csv&id=1n-zMS9Al94CPj_Tc3K7Adin-tN9x1RSjjx2UzJ4SV7Q&gid=590763272'
df  = pd.read_csv(url,
                  usecols=['New positive labs', 'New negative labs', 'Date'])
#df  = pd.read_csv('data/COVID-19 Rhode Island Data - COVID Trends.csv',
#                  usecols=['New positive labs', 'New negative labs', 'Date'])

df = df.rename(columns={'New positive labs': 'Positives', 'New negative labs': 'Negatives'})
df['Tests'] = df.Positives + df.Negatives
df = df.query('Tests>100')
df['date'] = pd.to_datetime(df['Date'])
df= df.set_index('date')
df['Date'] = pd.to_datetime(df['Date'])
df['Odds'] = df.Positives / df.Negatives
df = df[df['Date']>='2020-04-01']
df = df[df['Date']<'2020-05-18']
df_dict[dsname] = df

#https://governor.ri.gov/documents/orders/Executive-Order-20-04.pdf
bars = pd.to_datetime('03-16-2020 00:00', dayfirst=False)
# https://governor.ri.gov/documents/orders/Executive-Order-20-09.pdf
public_gatherings = pd.to_datetime('03-22-2020 00:00', dayfirst=False)

saty_at_home = pd.to_datetime('03-28-2020 00:00', dayfirst=False)
masks_cdc = pd.to_datetime('04-03-2020 00:00', dayfirst=False)
masks_at_work = pd.to_datetime('04-14-2020 00:00', dayfirst=False)
masks_public = pd.to_datetime('05-05-2020 00:00', dayfirst=False)
event_list = [bars, public_gatherings, saty_at_home, masks_cdc,masks_at_work, masks_public]
events[dsname] = event_list
```

## Virginia

```python
dsname = 'Virginia'
if not os.path.isfile('data/virginia.csv'):
    #df  = pd.read_csv('https://www.vdh.virginia.gov/content/uploads/sites/182/2020/05/VDH-COVID-19-PublicUseDataset-Tests_by-LabReportDate.csv')
    #df = pd.read_csv('data/VDH-COVID-19-PublicUseDataset-Tests_by-LabReportDate.csv')
    #df = df.rename(columns={'Number of Positive PCR Tests': 'Positives', 'Number of PCR Testing Encounters': 'Tests'})
    #df = df[df['Lab Report Date']!='Not Reported']
    #df['date'] = pd.to_datetime(df['Lab Report Date'])
    #df = df.groupby('date').sum()
    #df['Date'] = df.index
    dfs = []
    offset = 0
    while offset >=0:
        url = "https://data.virginia.gov/resource/3u5k-c2gr.csv/?$limit=5000&$offset={}".format(offset)
        df = pd.read_csv(url)
        dfs.append(df)
        if len(df)==5000:
            offset += 5000
        else:
            offset = -1
    df = pd.concat(dfs)
    df = df.rename(columns={'number_of_positive_pcr_testing': 'Positives', 'number_of_pcr_testing': 'Tests'})
    df = df[df['lab_report_date']!='Not Reported']
    df['date'] = pd.to_datetime(df['lab_report_date'])
    df = df.groupby('date').sum()
    df['Date'] = df.index
    #df.to_csv('data/virginia.csv')
else:
    df = pd.read_csv('data/virginia.csv', index_col='date', parse_dates=['date', 'Date'])
df = df[['Date', 'Tests', 'Positives']]
df['Negatives'] = df.Tests + df.Positives
df['Odds'] = df.Positives / df.Negatives
df = df[df['Date']>'2020-03-22']

# lab report day might be lagging
df = df[df['Date']<'2020-05-18']
#df = df[df['Date']<datetime.today()-timedelta(7)]
df_dict[dsname] = df
```

```python
dsname = 'Virginia'
# https://www.governor.virginia.gov/media/governorvirginiagov/executive-actions/EO-53-Temporary-Restrictions-Due-To-Novel-Coronavirus-(COVID-19).pdf
schools = pd.to_datetime('03-13-2020 00:00', dayfirst=False)
bars = pd.to_datetime('03-16-2020 00:00', dayfirst=False)
#
#public_gatherings = pd.to_datetime('03-22-2020 00:00', dayfirst=False)

saty_at_home = pd.to_datetime('04-02-2020 00:00', dayfirst=False)
masks_cdc = pd.to_datetime('04-03-2020 00:00', dayfirst=False)
# https://www.governor.virginia.gov/media/governorvirginiagov/executive-actions/EO-63-and-Order-Of-Public-Health-Emergency-Five---Requirement-To-Wear-Face-Covering-While-Inside-Buildings.pdf
masks_indoors = pd.to_datetime('05-29-2020 00:00', dayfirst=False)

event_list = [schools, bars, saty_at_home, masks_cdc, masks_indoors]
events[dsname] = event_list
```

```python
rows = []
for k, v in events.items():
    for d in v:
        rows.append({'State': k, 'Date': d.strftime('%d/%m/%Y'), 'Description': '', 'Type': ''})
events_df = pd.DataFrame(rows)#.set_index(['State', 'Date'])

events_df.loc[0, 'Description'] = 'Ban on large gatherings'
events_df.loc[1, 'Description'] = 'School closure'
events_df.loc[2, 'Description'] = 'Workforce up to 50%'
events_df.loc[3, 'Description'] = 'Stay at home order'
events_df.loc[4,'Description'] = 'CDC recommends masks'
events_df.loc[5,'Description'] = 'The state recommends masks for front line workers'
events_df.loc[6,'Description'] = 'The state recommends masks for the general public'
events_df.loc[0,'Type'] = 'Mobility restrictions'
events_df.loc[1,'Type'] = 'Mobility restrictions'
events_df.loc[2,'Type'] = 'Mobility restrictions'
events_df.loc[3,'Type'] = 'Mobility restrictions'
events_df.loc[4,'Type'] = 'Masks (CDC)'
events_df.loc[5,'Type'] = 'Masks (Local)'
events_df.loc[6,'Type'] = 'Masks (Local)'

events_df.loc[7, 'Description'] = 'Ban on large gatherings'
events_df.loc[8, 'Description'] = 'School closure'
events_df.loc[9, 'Description'] = 'Bars closure'
events_df.loc[10, 'Description'] = 'Malls closure'
events_df.loc[11, 'Description'] = 'Stay at home order'
events_df.loc[12, 'Description'] = 'CDC recommends masks'
events_df.loc[13, 'Description'] = 'The state recommends masks for the general public'
events_df.loc[7, 'Type'] = 'Mobility restrictions'
events_df.loc[8, 'Type'] = 'Mobility restrictions'
events_df.loc[9, 'Type'] = 'Mobility restrictions'
events_df.loc[10, 'Type'] = 'Mobility restrictions'
events_df.loc[11, 'Type'] = 'Mobility restrictions'
events_df.loc[12, 'Type'] = 'Masks (CDC)'
events_df.loc[13, 'Type'] = 'Masks (Local)'

events_df.loc[14, 'Description'] = 'School closure'
events_df.loc[15, 'Description'] = 'Stay at home order'
events_df.loc[16, 'Description'] = 'CDC recommends masks'
events_df.loc[17, 'Description'] = 'The state recommends masks for the general public'
events_df.loc[14, 'Type'] = 'Mobility restrictions'
events_df.loc[15, 'Type'] = 'Mobility restrictions'
events_df.loc[16, 'Type'] = 'Masks (CDC)'
events_df.loc[17, 'Type'] = 'Masks (Local)'

events_df.loc[18, 'Description'] = 'Stay at home order'
events_df.loc[19, 'Description'] = 'CDC recommends masks'
events_df.loc[20, 'Description'] = 'The state recommends masks in enclosed public spaces'
events_df.loc[18, 'Type'] = 'Mobility restrictions'
events_df.loc[19, 'Type'] = 'Masks (CDC)'
events_df.loc[20, 'Type'] = 'Masks (Local)'

events_df.loc[21, 'Description'] = 'Ban on large gatherings'
events_df.loc[22, 'Description'] = 'Close recreational establishments'
events_df.loc[23, 'Description'] = 'Stay at home order'
events_df.loc[24, 'Description'] = 'CDC recommends masks'
events_df.loc[25, 'Description'] = 'The state recommends masks for front line workers'
events_df.loc[26, 'Description'] = 'The state recommends masks for the general public'
events_df.loc[21, 'Type'] = 'Mobility restrictions'
events_df.loc[22, 'Type'] = 'Mobility restrictions'
events_df.loc[23, 'Type'] = 'Mobility restrictions'
events_df.loc[24, 'Type'] = 'Masks (CDC)'
events_df.loc[25, 'Type'] = 'Masks (Local)'
events_df.loc[26, 'Type'] = 'Masks (Local)'

events_df.loc[27, 'Description'] = 'Schools closure'
events_df.loc[28, 'Description'] = 'Ban on large gatherings'
events_df.loc[29, 'Description'] = 'Stay at home order'
events_df.loc[30, 'Description'] = 'CDC recommends masks'
events_df.loc[31, 'Description'] = 'The state recommends to wear masks indoors'

events_df.loc[27, 'Type'] = 'Mobility restrictions'
events_df.loc[28, 'Type'] = 'Mobility restrictions'
events_df.loc[29, 'Type'] = 'Mobility restrictions'
events_df.loc[30, 'Type'] = 'Masks (CDC)'
events_df.loc[31, 'Type'] = 'Masks (Local)'

#events_df['Date'] = events_df['Date'].strftime('%d/%m/%Y')
#formatters={"$R^2$": "{:0.3f}".format, 'fvalue':"{:0.3f}".format, 'f_pvalue': "{:.2e}".format},
out = events_df.to_latex(caption='Events by state and date.',
                         index=False,
                         bold_rows=True,
                         #escape=True,
                         label='tab:events'
                         )
print(out)
```

    \begin{table}
    \centering
    \caption{Events by state and date.}
    \label{tab:events}
    \begin{tabular}{llll}
    \toprule
             State &        Date &                                        Description &                   Type \\
    \midrule
          New York &  16/03/2020 &                            Ban on large gatherings &  Mobility restrictions \\
          New York &  18/03/2020 &                                     School closure &  Mobility restrictions \\
          New York &  20/03/2020 &                                Workforce up to 50\% &  Mobility restrictions \\
          New York &  22/03/2020 &                                 Stay at home order &  Mobility restrictions \\
          New York &  03/04/2020 &                               CDC recommends masks &            Masks (CDC) \\
          New York &  12/04/2020 &  The state recommends masks for front line workers &          Masks (Local) \\
          New York &  17/04/2020 &  The state recommends masks for the general public &          Masks (Local) \\
       Connecticut &  12/03/2020 &                            Ban on large gatherings &  Mobility restrictions \\
       Connecticut &  16/03/2020 &                                     School closure &  Mobility restrictions \\
       Connecticut &  17/03/2020 &                                       Bars closure &  Mobility restrictions \\
       Connecticut &  19/03/2020 &                                      Malls closure &  Mobility restrictions \\
       Connecticut &  23/03/2020 &                                 Stay at home order &  Mobility restrictions \\
       Connecticut &  03/04/2020 &                               CDC recommends masks &            Masks (CDC) \\
       Connecticut &  20/04/2020 &  The state recommends masks for the general public &          Masks (Local) \\
     Massachusetts &  22/03/2020 &                                     School closure &  Mobility restrictions \\
     Massachusetts &  24/03/2020 &                                 Stay at home order &  Mobility restrictions \\
     Massachusetts &  03/04/2020 &                               CDC recommends masks &            Masks (CDC) \\
     Massachusetts &  06/05/2020 &  The state recommends masks for the general public &          Masks (Local) \\
          Michigan &  24/03/2020 &                                 Stay at home order &  Mobility restrictions \\
          Michigan &  03/04/2020 &                               CDC recommends masks &            Masks (CDC) \\
          Michigan &  24/04/2020 &  The state recommends masks in enclosed public ... &          Masks (Local) \\
      Rhode Island &  16/03/2020 &                            Ban on large gatherings &  Mobility restrictions \\
      Rhode Island &  22/03/2020 &                  Close recreational establishments &  Mobility restrictions \\
      Rhode Island &  28/03/2020 &                                 Stay at home order &  Mobility restrictions \\
      Rhode Island &  03/04/2020 &                               CDC recommends masks &            Masks (CDC) \\
      Rhode Island &  14/04/2020 &  The state recommends masks for front line workers &          Masks (Local) \\
      Rhode Island &  05/05/2020 &  The state recommends masks for the general public &          Masks (Local) \\
          Virginia &  13/03/2020 &                                    Schools closure &  Mobility restrictions \\
          Virginia &  16/03/2020 &                            Ban on large gatherings &  Mobility restrictions \\
          Virginia &  02/04/2020 &                                 Stay at home order &  Mobility restrictions \\
          Virginia &  03/04/2020 &                               CDC recommends masks &            Masks (CDC) \\
          Virginia &  29/05/2020 &         The state recommends to wear masks indoors &          Masks (Local) \\
    \bottomrule
    \end{tabular}
    \end{table}

## Plots

I we plot the number of positive tests we can see that the data is noisy.
But, if we take into account the number of people tested each day, the data looks way more clean.

```python
plt.close(1)

current_palette = sns.color_palette()

df = pd.concat(df_dict).reset_index()
df = df.rename(columns={'level_0': 'State'})
df['Tests (right)'] = df['Tests']
g = sns.FacetGrid(df, col='State', col_wrap=2, xlim=[df.Date.min(), df.Date.max()], sharey=False, aspect=1.3, dropna=False)

legend_and_hand = []
def myplot(data, y=None, secondary_y=None, legend=None, color=None):
    ax = data.plot(x='Date', y=y[0], ax=plt.gca(), label=y[0])
    legend_and_hand.append(ax.get_legend_handles_labels())
    ax = data.plot(x='Date', y=y[1], secondary_y=secondary_y, ax=ax, label=y[1])
    legend_and_hand.append(ax.get_legend_handles_labels())
    return ax
g.map_dataframe(myplot, y=['Positives', 'Tests (right)'], secondary_y=['Tests (right)'])

#g.axes[0].xaxis.set_major_locator(locator)
#g.axes[0].xaxis.set_major_formatter(formatter)

g.add_legend({l[0]: h[0] for h,l in legend_and_hand[:2]}, bbox_to_anchor=(0.4, 0), ncol=2) #, loc='lower center'
g.set_titles('{col_name}')
plt.tight_layout()
plt.savefig('figs/state_testes.jpg', dpi=300)
```

## Relationship between the total number of infected individuals and positive tests

As has been shown previously [[1]](https://journals.plos.org/plosone/article?id=10.1371/journal.pone.0002185), the number of new infected individuals in a given day $k_t$ is given by:
$$
k_t = k_{t-1} e^{(R_{t-1}-1)\gamma}
$$

where $R_t$ is the effective reproductuve number and $\gamma^{-1}$ is the infectious period estimated as 7 days accoring to [2].

The following derivation was suggested to my by Will Meierjurgen Farr on this GitHub [Issue](https://github.com/k-sys/covid-19/issues/45#issuecomment-623782130).
Since we do not have access to the total number of infected indiviudals, but only to the population being tested, we have to use some statisticals assumtions on this populations.
If we asume that the people being tested, in a given day, is a sample of the population with COVID-19-like sympoms we can state that:

$$
n_{t} = [P_t(CV|sympoms) P_t(sympoms) +P_t(not CV|sympoms)P_t(sympoms)]Nf_t
$$

where $n_{t}$ is the number of people tested, $P_t(CV|sympoms)$ is the probablity of a pacient being positive for coronavirus given that the she is sympomatic, $P_t(sympoms)$ is the probablity of having COVID-like sympoms, $P_t(not CV|sympoms)$ is the probability of a pacient being coronavirus negative given he has COVID-19-like sympoms, $N$ is the total population, and $f_t$ is the fraction of people with sympoms that is selected to be tested (this number can be different each day, for example if the number of tests availabes changes).
Also, note that the probability of a test being positive in a given day is $Positive_t=P_t(CV|sympoms) P_t(sympoms) N f_t$

Now, if we assume that $P_t(sympoms|CV)=cte$ we can use Bayes theorem to show that:

$$
P_t(CV|sympoms) P_t(sympoms) \propto P_t(CV) = \frac{k_t}{N}
$$

Then:
$$
P_t(CV|sympoms) P_t(sympoms) \propto k_t
$$

Finally, if we assume that $P_t(not CV|sympoms)P_t(sympoms)=cte$:
$$
Odds_t = \frac{P(CV|sympoms) P(sympoms)Nf_t}{P_t(not CV|sympoms)P_t(sympoms)Nf_t}
$$
$$
Odds_t = \frac{P(CV|sympoms) P(sympoms)}{P_t(not CV|sympoms)P_t(sympoms)}
$$
$$
Odds_t \propto k_t
$$

\begin{equation}
Odds_t = Odds_{t-1} e^{(R_{t-1}-1)\gamma} (1)
\end{equation}

We used three hypothesis. First, constant population $N$ (for $P_t \propto k_t$ and for the evolution of $k_t$). Second, that the tested population is a random sample from the population with COVID19-like sympoms ($n_t = [P_t(CV|sympoms) P_t(sympoms) +P_t(not CV|sympoms)P_t(sympoms)]Nf_t$) this is not the case when people is being tested based on contacts for example. And third, that $P_t(not CV|sympoms)P_t(sympoms)=cte$, this is equivalent to say that the number of people with covid-19-like sympoms but without the coronavirus (for example people with the flu) is constant, or at least it changes are negligible compared with the changes in the amount of sympomatic people with coronavirus.

## Linearization

Defining

$$
b_i =  e^{(R_{i-1}-1)\gamma} (2)
$$

We can write [1] as:

\begin{equation}
odd_i = b_{i-1} * odd_{i-1} (3)
\end{equation}

Now, instead of using $b_i$ as the parameters to estimate we decompose each $b$ as follows:

$$
b_i = \prod_{j=0}^{i} a_j (4)
$$

Now, the $a_j$ represent the rate of change of the variable $b_i$. Next, we replace the [4] in [3]
$$
odd_i = \prod_{j=0}^{i-1} a_j * odd_{i-1}
$$
$$
odd_i = \prod_{j=0}^{i-1} a_j * \prod_{j=0}^{i-2} a_j * odd_{i-2}
$$
$$
odd_i = \prod_{k=0}^{i-1}\prod_{j=0}^{k} a_j * odd_{0}
$$
$$
odd_i = \prod_{j=0}^{i-1} a_j^{i-j} * odd_{0}
$$

Now, we liniarize this result and we generalize it to the case where $i=0$ using the $max$ function:

$$
log(odd_i) = \sum_{j=0}^{max(i-1, 0)} (i-j)log(a_j)  +  log(odd_{0}) (5)
$$

We can write [5] as a linear problem with the following definitions:

$$
y = X \beta + \beta_0
$$

$$
y_i = log(odd_i)
$$

$$
X_{i,j} =  max(i-j, 0)
$$

$$
\beta_i =  log(a_i) (6)
$$

Now if we apply a LASSO regression we will find the solution that minimize the following cost function

$$
Err = \sum (y-\hat{y})^2 + \alpha |\beta|_1
$$

Hopefully, this solution will be sparse which means that most of the $\beta_i$ will be $0$, and hence $a_i=1$.
This is equivalent to say that the $b_i$ are almost constant except at the values whete $a_i \neq 1$.

[1] Bettencourt, L. M. A., & Ribeiro, R. M. (2008). Real time bayesian estimation of the epidemic potential of emerging infectious diseases. PLoS ONE, 3(5). https://doi.org/10.1371/journal.pone.0002185

[2] Sanche, S., Lin, Y. T., Xu, C., Romero-Severson, E., Hengartner, N., & Ke, R. (2020). High Contagiousness and Rapid Spread of Severe Acute Respiratory Syndrome Coronavirus 2. Emerging Infectious Diseases, 26(7). https://doi.org/10.3201/eid2607.200282

## Classes

This cell contains the main class: LassoICSelector. Its main method is fit_best_alpha. It works as follows:
```
For each alpha value:
    1. Fits a lasso regression to the data
    2. Selectes the first non zero variable from each chunck
    3. Fits a linear regression with the selected variables
    4. Excludes all non sifgificative (p-value>0.05) variables and fits a linear model again

The linear model with less AIC from step 4 is selected.
```

```python
class FirstInChunkSelector(object):
    '''Selects first element from each non zero chunk.'''

    def __init__(self, clf):
        self.clf = clf
        self.coef = None
        self.mask = None

    def select_coef(self):
        n_features = len(self.clf.coef_)
        no_zero = np.zeros(n_features+1)
        no_zero[1:] = self.clf.coef_ != 0
        self.mask = np.diff(no_zero)>0
        self.mask[0] = True
        self.coef = self.clf.coef_[self.mask]
        return self.coef

    def transform(self, X):
        self.select_coef()
        return X[:, self.mask]

    def get_support(self):
        self.select_coef()
        return self.mask

    def get_number_of_features(self):
        self.select_coef()
        return sum(self.mask)

class LassoICSelector(object):
    """LASSO regression with FirstInChunk selector."""

    def __init__(self, X, y, criterion, alpha=0.05):
        self.lasso = linear_model.LassoLars(alpha=0, max_iter=100000)
        self.criterion = criterion
        self.selector = FirstInChunkSelector(self.lasso)
        self.OLS = sm.OLS
        self.ols = self.OLS(y, X)
        self.ols_results = None
        self.X = X
        self.y = y
        self.final_ols = False
        self.alpha = alpha

    def transform_to_ols(self, X):
        '''Selects only the features of X are used by OLS.
        Also, adds a coloumn with ones for the intercept.
        '''

        X_new = self.selector.transform(X)
        if self.final_ols:
            X_new = X[:, self.support]
        X_new_with_cte = np.hstack([X_new, np.ones((X_new.shape[0], 1))])
        return X_new_with_cte

    def fit(self, X, y):
        '''Selects features and fits the OLS.'''

        # select features
        X_new = self.transform_to_ols(X)

        # fit ols
        self.ols = self.OLS(y, X_new)
        self.ols_results = self.ols.fit()

        # iteratively remove non signicative variables and fit again
        mask = self.ols_results.pvalues < self.alpha / len(self.ols_results.pvalues)
        mask[0] = True
        Xnew = self.transform_to_ols(X)
        Xnew = Xnew[:, mask]
        self.support = self.selector.get_support()
        self.ols = self.OLS(y, Xnew)
        self.ols_results = self.ols.fit()
        while any(self.ols_results.pvalues[1:] >= self.alpha / len(self.ols_results.pvalues)):
            mask.values[mask.values] = (self.ols_results.pvalues < self.alpha / len(self.ols_results.pvalues)).values
            mask[0] = True
            Xnew = self.transform_to_ols(X)
            Xnew = Xnew[:, mask]
            self.support = self.selector.get_support()
            self.ols = self.OLS(y, Xnew)
            self.ols_results = self.ols.fit()

        self.support[self.support] = mask[:-1]

    def fit_best_alpha(self, X, y):
        '''returns the model with the lowst cirterion.'''

        self.lasso.fit(X, y)
        alphas = self.lasso.alphas_
        self.criterions_ = np.zeros(len(alphas))
        self.log_liklehods = np.zeros(len(alphas))

        for i, alpha in enumerate(alphas):
            self.lasso.coef_ = self.lasso.coef_path_[:, i]
            self.fit(X, y)
            self.criterions_[i], self.log_liklehods[i] = self.get_criterion(self.ols.exog, y)

        # we use a list of tuples to find the minimum cirterion value.
        # If there are ties, we use the maximum alpha value.
        criterions_idx = list(zip(self.criterions_, alphas, range(len(alphas))))
        criterion, alpha, idx = min(criterions_idx, key=lambda x: (x[0], -x[1]))
        #print(list(enumerate(zip(alphas, self.criterions_))))
        #print(alpha, criterion)
        self.lasso.coef_ = self.lasso.coef_path_[:, idx]
        self.lasso.alpha = alpha
        self.fit(X, y)
        self.final_ols = True

    def predict(self, X):
        '''Predicts y useing the OLS fit.'''

        return self.ols.predict(self.ols_results.params, X)

    def log_liklihood(self, X, y):
        '''Computes the log liklihood assuming normally distributed errors.'''

        eps64 = np.finfo('float64').eps

        # residuals
        R = y - self.predict(X)
        sigma2 = np.var(R)

        loglike = -0.5 * len(R) * np.log(sigma2)
        loglike -= 0.5 * len(R) * np.log(2*np.pi) - 0.5*len(R) + 0.5
        return loglike

    def get_criterion(self, X, y):
        '''Computes AIC or BIC criterion.'''

        n_samples = X.shape[0]
        if self.criterion == 'aic':
            K = 2  # AIC
        elif self.criterion == 'bic':
            K = np.log(n_samples)
        else:
            raise ValueError('criterion should be either bic or aic')

        log_like = self.log_liklihood(X, y)
        df = X.shape[1]

        aic = K * df - 2*log_like
        self.criterion_ = aic

        return self.criterion_, log_like
```

## Fit
Now, we create the linear system and fit the model

```python
plt.close('all')
lics_dict = {}
#fig, axes = plt.subplots(nrows=3, ncols=3, figsize=(12, 8))
gof_list = []
for i, state in enumerate(df.State.unique()):
    dfstate = df[df['State']==state]
    # create the independent and the dependent variables
    y = np.log(dfstate['Odds'])
    X = np.tri(len(y))
    X = np.cumsum(X, axis=0)[:, 1:]
    X = X[dfstate.Odds.notna(), :]
    y = y[dfstate.Odds.notna()]

    # create lasso instance
    lics = LassoICSelector(X, y.values, 'bic')

    # fit
    lics.fit_best_alpha(X, y)
    lics_dict[state] = lics

    #     ax = sns.lineplot(lics.lasso.alphas_, lics.criterions_, ax=axes[2, i])
    #     ax.vlines(lics.lasso.alpha, min(lics.criterions_), max(lics.criterions_))
    #     ax.set_ylabel('BIC')
    #     ax.set_xlabel('Alpha')
    #     ax.set_xscale('log')
    #     axes[0, i].plot(lics.lasso.alphas_, lics.lasso.coef_path_.T)
    #     axes[0, i].set_xscale('log')
    #     axes[0, i].set_title(state)
    #     axes[0, i].set_xlabel('Alpha')
    #     axes[0, i].set_ylabel('Coefficient Value')
    #     axes[0, i].vlines(lics.lasso.alpha, lics.lasso.coef_path_.min(), lics.lasso.coef_path_.max())
    #     axes[1, i].plot(lics.lasso.coef_)
    #     axes[1, i].scatter(np.arange(len(lics.lasso.coef_))[lics.support], lics.lasso.coef_[lics.support])
    #     axes[1, i].set_xlabel('Coefficient #')
    #     axes[1, i].set_ylabel('Coefficient Value')
    #     print(state)
    #     print('{} & {} & {} & {} & {} & {}'.format(state, lics.ols_results.rsquared, lics.ols_results.nobs, lics.ols_results.df_model, lics.ols_results.fvalue, lics.ols_results.f_pvalue))
    #     display(lics.ols_results.summary())
    gof_list.append({"State":state, '$R^2$':lics.ols_results.rsquared, "N": int(lics.ols_results.nobs), "df_model": int(lics.ols_results.df_model), "fvalue":lics.ols_results.fvalue, "f_pvalue":lics.ols_results.f_pvalue})
# plt.tight_layout(pad=0.0)
# plt.savefig('figs/framework.jpg', dpi=300)
# plt.show()
gof = pd.DataFrame(gof_list).sort_values('$R^2$', ascending=False)
out = gof.to_latex(index=False,formatters={"$R^2$": "{:0.3f}".format, 'fvalue':"{:0.3f}".format, 'f_pvalue': "{:.2e}".format},
             bold_rows=True, escape=False)
print(out)
```

    \begin{tabular}{lrrrrr}
    \toprule
             State & $R^2$ &   N &  df_model &   fvalue & f_pvalue \\
    \midrule
          New York & 0.989 &  62 &         4 & 1251.214 & 8.60e-55 \\
     Massachusetts & 0.982 &  64 &         3 & 1072.881 & 4.64e-52 \\
          Michigan & 0.971 &  62 &         3 &  639.142 & 2.23e-44 \\
       Connecticut & 0.951 &  61 &         3 &  365.902 & 3.44e-37 \\
      Rhode Island & 0.838 &  47 &         2 &  113.879 & 4.02e-18 \\
          Virginia & 0.741 &  56 &         2 &   75.841 & 2.82e-16 \\
    \bottomrule
    \end{tabular}

Lets copy the fitted values to a dataframe, and calculate the parameters and erros of the model.

```python
data_list = []
Rt_dict = {}
for state in df.State.unique():
    print(state)
    lics = lics_dict[state]
    data = df[df['State']==state].copy()

    # yhat = lics.ols_results.fittedvalues
    y = np.log(data['Odds'])
    X = np.tri(len(y))
    X = np.cumsum(X, axis=0)[:, 1:]
    Xols = lics.transform_to_ols(X)
    yhat = lics.ols.predict(lics.ols_results.params, Xols)
    # from equation 5
    odds_hat = np.exp(yhat)

    # the error in yhat is
    # Xols = lics.transform_to_ols(X)
    (yhat_std, yhat_l, yhat_u) = wls_prediction_std(lics.ols_results, Xols)
    oddshat_l = np.exp(yhat-2*yhat_std)
    oddshat_u = np.exp(yhat+2*yhat_std)

    data.loc[:, 'odds_hat'] = odds_hat
    data.loc[:, 'oddshat_l'] = oddshat_l
    data.loc[:, 'oddshat_u'] = oddshat_u

    # use coefficients to calculate Rt
    coef = np.zeros(len(data))
    coef_std = np.zeros_like(coef) * np.nan
    ind = np.squeeze(np.argwhere(lics.support))

    # we do not use the last coefficient since it's the intercept (=log(odds_0))
    coef[ind] = lics.ols_results.params[:-1]

    # using equation 2, 4 and 6
    data.loc[:, 'R'] = np.cumsum(coef)/GAMMA+1

    # get covarinace matrix of coefficients
    cov = lics.ols_results.cov_params().values

    # since the values of Rts are a sum of variables, we use the formula
    # of the sum of gaussian variables with a known covariance matrix
    stds = [np.sqrt(cov[:n, :n].sum()) for n in range(1, cov.shape[0])]
    if len(stds)==1:
        stds = stds[0]
    coef_std[ind] = stds

    # error propagation formula
    data.loc[:, 'Rstd'] = coef_std / GAMMA

    data['Rstd'] = data['Rstd'].fillna(method='ffill')
    data['R_l'] = data['R'] - 2*data['Rstd']
    data['R_u'] = data['R'] + 2*data['Rstd']

    r_index = data.R.diff() != 0
    Rts = data.loc[r_index, ['Date', 'R', 'R_l', 'R_u']]
    Rts['delta'] = Rts['R_u'] - Rts['R_l']
    Rt_dict[state] = Rts
    display(Rts)
    #print(np.cumsum(coef)[r_index], coef_std[r_index])
    data_list.append(data)
data = pd.concat(data_list)
Rts = pd.concat(Rt_dict, names=['Dataset'])
```

    New York

      Date
      R
      R_l
      R_u
      delta

      0
      2020-03-15
      2.097828
      1.943739
      2.251917
      0.308179

      8
      2020-03-23
      1.567515
      1.431863
      1.703166
      0.271303

      15
      2020-03-30
      0.726109
      0.675574
      0.776644
      0.101070

      30
      2020-04-14
      0.439509
      0.415892
      0.463126
      0.047234

    Connecticut

      Date
      R
      R_l
      R_u
      delta

      62
      2020-03-20
      1.779764
      1.596979
      1.962549
      0.365570

      71
      2020-03-29
      0.981041
      0.905938
      1.056145
      0.150207

      86
      2020-04-13
      0.601700
      0.571878
      0.631522
      0.059644

    Massachusetts

      Date
      R
      R_l
      R_u
      delta

      123
      2020-03-15
      1.872406
      1.814039
      1.930773
      0.116734

      136
      2020-03-28
      1.158793
      1.122085
      1.195501
      0.073416

      151
      2020-04-12
      0.659066
      0.643282
      0.674850
      0.031568

    Michigan

      Date
      R
      R_l
      R_u
      delta

      187
      2020-03-17
      1.464358
      1.393495
      1.535221
      0.141726

      205
      2020-04-04
      0.415219
      0.368090
      0.462348
      0.094258

      227
      2020-04-26
      0.670931
      0.612907
      0.728954
      0.116047

    Rhode Island

      Date
      R
      R_l
      R_u
      delta

      249
      2020-04-01
      1.070697
      1.005958
      1.135437
      0.129479

      268
      2020-04-20
      0.714994
      0.672172
      0.757816
      0.085644

    Virginia

      Date
      R
      R_l
      R_u
      delta

      296
      2020-03-23
      1.130957
      1.108889
      1.153025
      0.044135

      323
      2020-04-19
      0.896946
      0.875805
      0.918088
      0.042283

```python
Rts = pd.concat(Rt_dict, names=['Dataset'])
Rts = Rts.reset_index().drop('level_1', axis=1)#[['Dataset', 'Date', 'R', #.rename(columns={'level_'})
'('+Rts.R_l.round(2).astype(str)

Rts['figures'] = (-np.floor(np.log10(Rts.delta))+1).astype(int)
Rts['R'] = Rts.apply(lambda row: np.round(row['R'], row['figures']), axis=1)
Rts['R'] = Rts.apply(lambda row: '{:0.{}f}'.format(row['R'], row['figures']), axis=1)
Rts['R_l'] = Rts.apply(lambda row: np.round(row['R_l'], row['figures']), axis=1)
Rts['R_u'] = Rts.apply(lambda row: np.round(row['R_u'], row['figures']), axis=1)
Rts['95 % C.I.'] = Rts.apply(lambda row: '({}, {})'.format(row['R_l'], row['R_u']), axis=1)
print(Rts[['Dataset', 'Date', 'R', '95 % C.I.']].to_latex(caption='Values of $R_t$ for each dataset and times of $R_t$ change.',
                                                         label='tab:rt', index=False))
```

    \begin{table}
    \centering
    \caption{Values of $R_t$ for each dataset and times of $R_t$ change.}
    \label{tab:rt}
    \begin{tabular}{llll}
    \toprule
           Dataset &       Date &      R &       95 \% C.I. \\
    \midrule
          New York & 2020-03-15 &   2.10 &    (1.94, 2.25) \\
          New York & 2020-03-23 &   1.57 &     (1.43, 1.7) \\
          New York & 2020-03-30 &   0.73 &    (0.68, 0.78) \\
          New York & 2020-04-14 &  0.440 &  (0.416, 0.463) \\
       Connecticut & 2020-03-20 &   1.78 &     (1.6, 1.96) \\
       Connecticut & 2020-03-29 &   0.98 &    (0.91, 1.06) \\
       Connecticut & 2020-04-13 &  0.602 &  (0.572, 0.632) \\
     Massachusetts & 2020-03-15 &   1.87 &    (1.81, 1.93) \\
     Massachusetts & 2020-03-28 &  1.159 &  (1.122, 1.196) \\
     Massachusetts & 2020-04-12 &  0.659 &  (0.643, 0.675) \\
          Michigan & 2020-03-17 &   1.46 &    (1.39, 1.54) \\
          Michigan & 2020-04-04 &  0.415 &  (0.368, 0.462) \\
          Michigan & 2020-04-26 &   0.67 &    (0.61, 0.73) \\
      Rhode Island & 2020-04-01 &   1.07 &    (1.01, 1.14) \\
      Rhode Island & 2020-04-20 &  0.715 &  (0.672, 0.758) \\
          Virginia & 2020-03-23 &  1.131 &  (1.109, 1.153) \\
          Virginia & 2020-04-19 &  0.897 &  (0.876, 0.918) \\
    \bottomrule
    \end{tabular}
    \end{table}

```python
Rtsdiff = Rts.copy()
Rtsdiff = Rtsdiff.drop(1)
Rtsdiff = Rtsdiff[Rtsdiff.index<10]
Rtsdiff['R2'] = Rtsdiff['R'].astype(float)
Rtsdiff['R1'] = Rtsdiff.groupby('Dataset')['R2'].shift(1)
Rtsdiff['Rd'] = 1-Rtsdiff['R2']/Rtsdiff['R1']

Rtsdiff['Delta_R1'] = Rtsdiff.groupby('Dataset')['delta'].shift(1)
Rtsdiff['Delta_R2'] = Rtsdiff['delta']
Rtsdiff['Delta_Rd'] = Rtsdiff['Delta_R1']*Rtsdiff['R2']/Rtsdiff['R1']**2 + Rtsdiff['Delta_R2']/Rtsdiff['R1']
Rtsdiff = Rtsdiff.dropna()
Rtsdiff['Rd_l'] = Rtsdiff['Rd'] - Rtsdiff['Delta_Rd']/2
Rtsdiff['Rd_u'] = Rtsdiff['Rd'] + Rtsdiff['Delta_Rd']/2

Rtsdiff['Rd'] = Rtsdiff['Rd']*100
Rtsdiff['Rd_l'] = Rtsdiff['Rd_l']*100
Rtsdiff['Rd_u'] = Rtsdiff['Rd_u']*100
Rtsdiff['figures'] = (-np.floor(np.log10(100*Rtsdiff.Delta_Rd))+1).astype(int)
Rtsdiff['Rd'] = Rtsdiff.apply(lambda row: np.round(row['Rd'], row['figures']), axis=1)
Rtsdiff['Rd'] = Rtsdiff.apply(lambda row: '{:0.{}f}'.format(row['Rd'], row['figures']), axis=1)
Rtsdiff['Rd_l'] = Rtsdiff.apply(lambda row: np.round(row['Rd_l'], row['figures']), axis=1)
Rtsdiff['Rd_u'] = Rtsdiff.apply(lambda row: np.round(row['Rd_u'], row['figures']), axis=1)
Rtsdiff['95 \% C.I.'] = Rtsdiff.apply(lambda row: '({}, {})'.format(row['Rd_l'], row['Rd_u']), axis=1)

Rtsdiff = Rtsdiff.reset_index()
Rtsdiff['Intervention'] = ['Mobility restrictions', 'Masks',
                           'Mobility restrictions', 'Masks',
                           'Mobility restrictions', 'Masks']
#Rtsdiff[['Dataset', 'Rd','95 % C.I.', 'Intervention']]
Rtsdiff['$R_t$ \% reduction'] = Rtsdiff['Rd']
Rtsdiff = Rtsdiff[['Dataset', 'Intervention', '$R_t$ \% reduction', '95 \% C.I.']]
print(Rtsdiff.to_latex(caption='Changes in $R_t$ due to each type of NPIs.',
                       label='tab:rt_reduction', index=False, escape=False))
```

    \begin{table}
    \centering
    \caption{Changes in $R_t$ due to each type of NPIs.}
    \label{tab:rt_reduction}
    \begin{tabular}{llll}
    \toprule
           Dataset &           Intervention & $R_t$ \% reduction &    95 \% C.I. \\
    \midrule
          New York &  Mobility restrictions &               65.2 &  (60.3, 70.2) \\
          New York &                  Masks &                 40 &  (32.0, 47.0) \\
       Connecticut &  Mobility restrictions &                 45 &  (35.0, 55.0) \\
       Connecticut &                  Masks &                 39 &  (31.0, 46.0) \\
     Massachusetts &  Mobility restrictions &               38.0 &  (34.1, 41.9) \\
     Massachusetts &                  Masks &               43.1 &  (40.0, 46.3) \\
    \bottomrule
    \end{tabular}
    \end{table}

```python
state = 'New York'
params_stats_list = []
for state in lics_dict:
    res = lics_dict[state].ols_results
    dir(res)
    params_stats = res.params
    params_stats = pd.DataFrame({'Coef. Name': res.params.index,
                                 'Coefficient':res.params,
                                 'Std': pd.Series(np.diag(res.cov_params())**0.5, index=res.cov_params().index),
                                  'p-value': res.pvalues},
                                index=res.params.index)
    params_stats['c_l'] = params_stats['Coefficient']-2*params_stats['Std']
    params_stats['c_u'] = params_stats['Coefficient']+2*params_stats['Std']
    params_stats['delta'] = 4*params_stats['Std']
    params_stats['figures'] = (-np.floor(np.log10(params_stats.delta))+1).astype(int)
    params_stats['c_u'] = params_stats.apply(lambda row: np.round(row['c_u'], row['figures']), axis=1)
    params_stats['c_l'] = params_stats.apply(lambda row: np.round(row['c_l'], row['figures']), axis=1)
    params_stats['Coefficient'] = params_stats.apply(lambda row: np.round(row['Coefficient'], row['figures']), axis=1)
    params_stats['Coefficient'] = params_stats.apply(lambda row: '${:0.{}f}$'.format(row['Coefficient'], row['figures']), axis=1)
    params_stats['c_u'] = params_stats.apply(lambda row: '{:0.{}f}'.format(row['c_u'], row['figures']), axis=1)
    params_stats['c_l'] = params_stats.apply(lambda row: '{:0.{}f}'.format(row['c_l'], row['figures']), axis=1)
    params_stats['95\% C.I.'] = params_stats.apply(lambda row: '$({}, {})$'.format(row['c_l'], row['c_u']), axis=1)
    params_stats['Dataset'] = state
    params_stats_list.append(params_stats)

params_stats = pd.concat(params_stats_list)[['Dataset', 'Coef. Name', 'Coefficient', '95\% C.I.', 'p-value']]
params_stats_latex = params_stats.to_latex(index=False,
                                           label='tab:coef',
                                           caption='Parameter values and statistics for the selected model for each dataset',
                                           escape=False,
                                           formatters={'p-value': "{:.2e}".format})
print(params_stats_latex)
```

    \begin{table}
    \centering
    \caption{Parameter values and statistics for the selected model for each dataset}
    \label{tab:coef}
    \begin{tabular}{llllr}
    \toprule
           Dataset & Coef. Name & Coefficient &           95\% C.I. &  p-value \\
    \midrule
          New York &         x1 &     $0.146$ &    $(0.126, 0.167)$ & 1.94e-20 \\
          New York &         x2 &    $-0.071$ &  $(-0.106, -0.036)$ & 1.60e-04 \\
          New York &         x3 &    $-0.112$ &  $(-0.135, -0.089)$ & 7.51e-14 \\
          New York &         x4 &    $-0.038$ &  $(-0.047, -0.029)$ & 8.96e-12 \\
          New York &      const &     $-1.70$ &    $(-1.81, -1.59)$ & 4.50e-37 \\
       Connecticut &         x1 &     $0.104$ &    $(0.080, 0.128)$ & 9.05e-12 \\
       Connecticut &         x2 &    $-0.106$ &  $(-0.138, -0.075)$ & 7.91e-09 \\
       Connecticut &         x3 &    $-0.051$ &  $(-0.063, -0.038)$ & 8.56e-11 \\
       Connecticut &      const &     $-1.35$ &    $(-1.51, -1.19)$ & 9.49e-24 \\
     Massachusetts &         x1 &     $0.116$ &    $(0.109, 0.124)$ & 9.72e-38 \\
     Massachusetts &         x2 &    $-0.095$ &  $(-0.107, -0.084)$ & 5.14e-24 \\
     Massachusetts &         x3 &    $-0.067$ &  $(-0.073, -0.060)$ & 2.86e-29 \\
     Massachusetts &      const &     $-2.63$ &    $(-2.70, -2.56)$ & 1.07e-60 \\
          Michigan &         x1 &     $0.062$ &    $(0.052, 0.071)$ & 5.51e-19 \\
          Michigan &         x2 &    $-0.140$ &  $(-0.154, -0.126)$ & 2.03e-27 \\
          Michigan &         x3 &     $0.034$ &    $(0.022, 0.047)$ & 1.22e-06 \\
          Michigan &      const &     $-1.42$ &    $(-1.54, -1.30)$ & 1.67e-31 \\
      Rhode Island &         x1 &     $0.009$ &    $(0.001, 0.018)$ & 3.43e-02 \\
      Rhode Island &         x2 &    $-0.047$ &  $(-0.060, -0.035)$ & 3.29e-09 \\
      Rhode Island &      const &     $-1.93$ &    $(-2.04, -1.81)$ & 9.00e-33 \\
          Virginia &         x1 &    $0.0175$ &  $(0.0145, 0.0204)$ & 1.51e-16 \\
          Virginia &         x2 &    $-0.031$ &  $(-0.036, -0.026)$ & 6.87e-17 \\
          Virginia &      const &     $-2.22$ &    $(-2.27, -2.16)$ & 1.87e-57 \\
    \bottomrule
    \end{tabular}
    \end{table}

Now, we plot the Rt as function of time and the fitted values.

```python
plt.close('all')

g1 = sns.FacetGrid(data.set_index('date'), col='State', col_wrap=2, ylim=[0, 2.5], aspect=1.5, dropna=False, col_order=gof.State)
def myplot(data, y=None, color=None):
    data.plot(x='Date', y=y, ax=plt.gca())
#g1.map_dataframe(sns.lineplot, x='Date', y='R')
g1.map_dataframe(myplot, y='R')
g1.map(plt.fill_between, 'Date', 'R_u', 'R_l', alpha=0.2)
#ax = data.plot(x='Date', y='R', legend=False)
lines = []
for ax, dsname in zip(g1.axes, gof.State):
    if dsname in events:
        for line in events[dsname]:
            if line < pd.to_datetime('03-04-2020 00:00', dayfirst=True):
                color = current_palette[1]
                label = 'Mobility restrictions'
            elif line == pd.to_datetime('03-04-2020 00:00', dayfirst=True):
                color = 'k'
                label = 'Masks (CDC)'
            else:
                label = 'Masks (State)'
                color = current_palette[2]
            lines.append(ax.axvline(line, 0,1, linestyle='--', color=color, label=label))

#     lines_mr = [line for line in events[dsname] if line<pd.to_datetime('03-04-2020 00:00', dayfirst=True)]
#     lines_masks = [line for line in events[dsname] if line>pd.to_datetime('03-04-2020 00:00', dayfirst=True)]
#     ax.axvlines(lines_mr, 0,1, linestyle='--', color='r', label='Movility restrictions')
#     ax.axvline(pd.to_datetime('03-04-2020 00:00', dayfirst=True), 0,1, linestyle='--', color='r', label='Masks (CDC)')
#     ax.axvlines(lines_masks, 0,1, linestyle='--', color='r', label='Masks (Local)')

g1.set(ylabel='$R_t$')
#from pandas.plotting._matplotlib.timeseries import format_dateaxis
#format_dateaxis(g1.axes[0,0], 1, data.index)
#g1.axes[0,0].xaxis.set_major_locator(locator)
#g1.axes[0,0].xaxis.set_major_formatter(formatter)
g1.set(yscale='linear')
g1.axes[0].legend([lines[0], lines[4], lines[5]], ['Mobility restrictions', 'Masks (CDC)', 'Masks (Local)'])
#print(g1.axes[0, 0].legend().get_legend_handler_map())
plt.xlabel('')

g1.set(xlim=[data.date.min(), data.date.max()])
g1.set_titles('{col_name}')
plt.tight_layout()
plt.savefig('figs/states6_RtL1.jpg', dpi=300)

stacked = data.set_index(['Date', 'State'])[['Odds', 'odds_hat', 'oddshat_u', 'oddshat_l']]
stacked = stacked.stack().reset_index().rename(columns={'level_2': 'variable', 0: 'val'})
stacked['Branch'] = 'Fit'
stacked.loc[stacked['variable'] == 'Odds', 'Branch'] = 'Data'
stacked = stacked.set_index(['Date', 'State', 'Branch', 'variable']).unstack().reset_index()
stacked.columns = ['Date', 'State', 'Branch', 'Odds', 'odds_hat', 'oddshat_u', 'oddshat_l']
stacked = stacked.sort_values(['State', 'Date'])

g = sns.FacetGrid(stacked, col='State', col_wrap=2, hue='Branch', aspect=1.3, dropna=False, hue_order=['Data', 'Fit'], col_order=gof.State)
g.map_dataframe(sns.lineplot, x='Date', y='odds_hat')
g.map_dataframe(sns.scatterplot, x='Date', y='Odds')
g.map(plt.fill_between, 'Date', 'oddshat_l', 'oddshat_u', alpha=0.1)
#.set_yscale('log')
g.set(yscale='log')

hl = g.axes[0].get_legend_handles_labels()

handlers = [hl[0][1], hl[0][0]]
labels = [hl[1][1], hl[1][0]]

lh = {l:h for h,l in zip(handlers, labels)}
# plt.ylabel('Odds')
g.set(xlim = (data['Date'].min(), data['Date'].max()))
g.set_titles('{col_name}')
g.set(yscale='log')
g.axes[0].xaxis.set_major_locator(locator)
g.axes[0].xaxis.set_major_formatter(formatter)
g.add_legend(lh, bbox_to_anchor=(0.6, 0), title='', ncol=3)#, loc='lower center'

g.set(ylabel='Odds')
plt.tight_layout()
plt.savefig('figs/states6_OddsL1.jpg', dpi=300)
```

## Counterfactual

```python
# for state in df.State.unique():
#     print(state)
#     lics = lics_dict[state]
#     #data = df[df['State']==state].copy()

#     # yhat = lics.ols_results.fittedvalues
#     y = np.log(data.loc[data['State']==state, 'Odds'])
#     X = np.tri(len(y))
#     X = np.cumsum(X, axis=0)[:, 1:]
#     fitted_params = lics.ols_results.params.copy()
#     current_palette = sns.color_palette()

#     params_cf = lics.ols_results.params.copy()
#     params_cf[-2] = 0
#     lics.ols_results.params = params_cf
#     #exog = np.hstack([lics.transform_to_ols(X)[:,:-2], lics.transform_to_ols(X)[:,-1:]])
#     yhat = lics.predict(lics.transform_to_ols(X))
#     odds_cf = np.exp(yhat)
#     (yhat_std, yhat_l, yhat_u) = wls_prediction_std(lics.ols_results, lics.transform_to_ols(X))
#     oddshat_std = odds_cf*yhat_std

#     lics.ols_results.params = fitted_params
#     ratio_cf = odds_cf / (odds_cf+1)

#     coef = np.zeros(len(data.loc[data['State']==state]))
#     ind = np.squeeze(np.argwhere(lics.support))
#     coef[ind] = params_cf[:-1]
#     data.loc[data['State']==state, 'R_cf'] = np.cumsum(coef)/GAMMA+1

#     data.loc[data['State']==state, 'Odds CF'] = odds_cf
#     data.loc[data['State']==state, 'Odds CF std'] = oddshat_std
#     data.loc[data['State']==state, 'oddshat_l_cf'] = odds_cf - 2*oddshat_std
#     data.loc[data['State']==state, 'oddshat_u_cf'] = odds_cf + 2*oddshat_std

#     data.loc[data['State']==state, 'ratio_cf'] = ratio_cf
#     break

# #     plt.figure()
# #     ax = sns.scatterplot(x='Date', y='Odds', data=data[data['State']==state], label='Data', c=np.array([current_palette[0]]))

# #     ax = sns.lineplot(x='Date', y='odds_hat', data=data[data['State']==state], ax=ax, palette=[current_palette[i]], label='Fit')
# #     ax.fill_between(data.loc[data['State']==state, 'Date'], data.loc[data['State']==state, 'oddshat_l'],
# #                     data.loc[data['State']==state, 'oddshat_u'],
# #                     alpha=0.1)

# #     ax = sns.lineplot(x='Date', y='Odds CF', data=data[data['State']==state], ax=ax, label='Counterfactual No Masks')#, palette=[current_palette[i]]
# #     ax.fill_between(data.loc[data['State']==state, 'Date'], data.loc[data['State']==state, 'oddshat_l_cf'],
# #                     data.loc[data['State']==state, 'oddshat_u_cf'],
# #                     alpha=0.1)

# #     plt.yscale('log')

# #     ax.xaxis.set_major_locator(locator)
# #     ax.xaxis.set_major_formatter(formatter)
# #     plt.grid(True)
# #     plt.ylabel('Odds')
# #     ax.set_xlim(data['Date'].min(), data.loc[data.odds_hat.notna(), 'Date'].max())
# #     plt.tight_layout()
# #     plt.savefig('figs/odds_cf_masks.jpg', dpi=300)
# #     plt.show()
```

```python
for state in df.State.unique():
    print(state)
    lics = lics_dict[state]

    y = np.log(data.loc[data['State']==state, 'Odds'])
    X = np.tri(len(y))
    X = np.cumsum(X, axis=0)[:, 1:]
    exog_full = lics.transform_to_ols(X)
    ind = exog_full[:, -2]<2
    y_sub = y[ind]
    exog_sub = exog_full[ind,:]
    exog_sub = np.hstack([exog_sub[:, :-2],exog_sub[:,-1:]])
    ols = sm.OLS(y_sub, exog_sub)
    res = ols.fit()
    exog = np.hstack([exog_full[:, :-2],exog_full[:,-1:]])
    yhat = res.predict(exog)
    yhat_orig = lics.predict(exog_full)

    odds_cf = np.exp(yhat)
    (yhat_std, yhat_l, yhat_u) = wls_prediction_std(res, exog)
    oddshat_std = odds_cf*yhat_std

    #lics.ols_results.params = fitted_params
    ratio_cf = odds_cf / (odds_cf+1)

    #ind = np.squeeze(np.argwhere(lics.support))

    data.loc[data['State']==state, 'Odds CF'] = odds_cf
    data.loc[data['State']==state, 'Odds CF std'] = oddshat_std
    data.loc[data['State']==state, 'oddshat_l_cf'] = odds_cf - 2*oddshat_std
    data.loc[data['State']==state, 'oddshat_u_cf'] = odds_cf + 2*oddshat_std

    data.loc[data['State']==state, 'ratio_cf'] = ratio_cf
```

    New York
    Connecticut
    Massachusetts
    Michigan
    Rhode Island
    Virginia

```python
# plt.close('all')
# g1 = sns.FacetGrid(data, row='State', aspect=1.5)
# g1.map_dataframe(sns.scatterplot, x='Date', y='Odds', label='Data')
# g1.map_dataframe(sns.lineplot, x='Date', y='odds_hat', label='Fit')
# def myfill_between(*args, **kwargs):
#     ax = plt.gca()
#     ax.fill_between(*args, **kwargs)
# g1.map(myfill_between, 'Date', 'oddshat_l', 'oddshat_u', alpha=0.1)

# g1.map_dataframe(sns.lineplot, x='Date', y='Odds CF', label='Counterfactual No Masks', palette=[current_palette[2]])
# g1.map(plt.fill_between, 'Date', 'oddshat_l_cf', 'oddshat_u_cf', alpha=0.1)

# plt.yscale('log')

# #ax.xaxis.set_major_locator(locator)
# #ax.xaxis.set_major_formatter(formatter)
# #plt.grid(True)
# #plt.ylabel('Odds')
# g1.set(xlim=[data['Date'].min(), data.loc[data.odds_hat.notna(), 'Date'].max()])
# #plt.tight_layout()
# #plt.savefig('figs/odds_cf_masks.jpg', dpi=300)
# plt.show()
```

```python
pepe = data.set_index(['Date', 'State', ])[['Odds', 'odds_hat', 'oddshat_u', 'oddshat_l', 'Odds CF', 'oddshat_u_cf', 'oddshat_l_cf']]
pepe = pepe.stack().reset_index().rename(columns={'level_2': 'variable', 0: 'val'})
pepe['Branch'] = 'Fit'
pepe.loc[pepe['variable'].str.lower().str.contains('cf'), 'Branch'] = 'Counterfactual'
pepe.loc[pepe['variable'] == 'Odds', 'Branch'] = 'Data'
pepe.loc[pepe['variable']=='Odds CF', 'variable'] = 'odds_hat'
pepe.loc[pepe['variable']=='oddshat_u_cf', 'variable'] = 'oddshat_u'
pepe.loc[pepe['variable']=='oddshat_l_cf', 'variable'] = 'oddshat_l'
pepe = pepe.set_index(['Date', 'State', 'Branch', 'variable']).unstack().reset_index()
pepe.columns = ['Date', 'State', 'Branch', 'Odds', 'odds_hat', 'oddshat_u', 'oddshat_l']

pepe = pepe[pepe['State'].isin(['New York', 'Connecticut', 'Massachusetts'])]
plt.close('all')

g1 = sns.FacetGrid(pepe, row='State', hue='Branch', aspect=1.5, dropna=False, hue_order=['Data', 'Fit', 'Counterfactual'])
legend_and_hand = []
def myplot(data, x=None, y=None, color=None, label=None):
    ax = plt.gca()
    ax = data.plot(x=x, y=y, ax=ax, label=label)
    legend_and_hand.append(ax.get_legend_handles_labels())
    return ax
g1.map_dataframe(myplot, x='Date', y='odds_hat')
g1.map_dataframe(sns.scatterplot, x='Date', y='Odds')

g1.map(plt.fill_between, 'Date', 'oddshat_l', 'oddshat_u', alpha=0.1)
hl = g1.axes[0,0].get_legend_handles_labels()

lh = {l:h for h,l in zip(hl[0][1:4], hl[1][1:4])}

g1.add_legend(lh, bbox_to_anchor=(0.7, 0.01), title='', ncol=3)#, loc='lower center'

plt.yscale('log')
plt.xlabel('')

g1.set(xlim=[data['Date'].min(), data.loc[data.odds_hat.notna(), 'Date'].max()], ylabel='Odds')
g1.set_titles("{row_name}")
plt.tight_layout(h_pad=1)
plt.savefig('figs/odds_cf_masks.jpg', dpi=300)
plt.show()
```

```python
for state in  ['New York', 'Connecticut', 'Massachusetts']:
    subdata = data[data['State']==state]
    Positives_CF = subdata['Tests'] * subdata['Odds CF']/(subdata['Odds CF']+1)
    Positives_CF_std = subdata['Odds CF std'] * subdata['Tests'] /(subdata['Odds CF']+1)**2
    Positives_CF_sum_std = np.sqrt((Positives_CF_std**2).sum())
    Positive_diff = (Positives_CF - subdata['Positives']).sum()
    Positive_diff_l = Positive_diff - 2* Positives_CF_sum_std
    Positive_diff_u = Positive_diff + 2* Positives_CF_sum_std
    Positives_fit = subdata['Tests'] * subdata['odds_hat']/(subdata['odds_hat']+1)
    print('{} & Counterfactual cases diff {} & CI ({} {})'.format(state, Positive_diff, Positive_diff_l, Positive_diff_u))
    #print('{} & Counterfactual death diff {} CI ({} {})'.format(0.077*Positive_diff, 0.077*Positive_diff_l, 0.077*Positive_diff_u))

Positives_CF = data['Tests'] * data['Odds CF']/(data['Odds CF']+1)
Positives_CF_std = data['Odds CF std'] * data['Tests'] /(data['Odds CF']+1)**2
Positives_CF_sum_std = np.sqrt((Positives_CF_std**2).sum())
Positive_diff = (Positives_CF - data['Positives']).sum()
Positive_diff_l = Positive_diff - 2* Positives_CF_sum_std
Positive_diff_u = Positive_diff + 2* Positives_CF_sum_std
Positives_fit = data['Tests'] * data['odds_hat']/(data['odds_hat']+1)
print('All & Counterfactual cases diff {} &  CI ({} {})'.format(Positive_diff, Positive_diff_l, Positive_diff_u))
print('Counterfactual death diff {} CI ({} {})'.format(0.077*Positive_diff, 0.077*Positive_diff_l, 0.077*Positive_diff_u))
```

    New York & Counterfactual cases diff 76939.6040468762 & CI (65252.762298588874 88626.44579516351)
    Connecticut & Counterfactual cases diff 36354.90071094035 & CI (33124.178817701126 39585.62260417957)
    Massachusetts & Counterfactual cases diff 83143.63343675967 & CI (79567.02602730246 86720.24084621688)
    All & Counterfactual cases diff 200736.92764519624 &  CI (187810.77529641215 213663.07999398033)
    Counterfactual death diff 15456.74342868011 CI (14461.429697823734 16452.057159536485)

```python
max_date = min(data.loc[data['State']=='New York', 'Date'].max(), data.loc[data['State']=='Connecticut', 'Date'].max(), data.loc[data['State']=='Massachusetts', 'Date'].max())
min_date = pd.to_datetime('2020-04-14')
for state in  ['New York', 'Connecticut', 'Massachusetts']:
    subdata = data.loc[(data['State']==state) & (data['Date']<=max_date) & (data['Date']>=min_date)]
    Positives_CF = subdata['Tests'] * subdata['Odds CF']/(subdata['Odds CF']+1)
    Positives_CF_std = subdata['Odds CF std'] * subdata['Tests'] /(subdata['Odds CF']+1)**2
    Positives_CF_sum_std = np.sqrt((Positives_CF_std**2).sum())
    Positive_diff = (Positives_CF - subdata['Positives']).sum()
    Positive_diff_l = Positive_diff - 2* Positives_CF_sum_std
    Positive_diff_u = Positive_diff + 2* Positives_CF_sum_std
    Positives_fit = subdata['Tests'] * subdata['odds_hat']/(subdata['odds_hat']+1)
    print('{} & Counterfactual cases diff {} & CI ({} {})'.format(state, Positive_diff, Positive_diff_l, Positive_diff_u))

ind = (data['Date']<=max_date) & (data['Date']>=min_date)
Positives_CF = data.loc[ind, 'Tests'] * data.loc[ind, 'Odds CF']/(data.loc[ind, 'Odds CF']+1)
Positives_CF_std = data.loc[ind, 'Odds CF std'] * data.loc[ind, 'Tests'] /(data.loc[ind, 'Odds CF']+1)**2
Positives_CF_sum_std = np.sqrt((Positives_CF_std**2).sum())
Positive_diff = (Positives_CF - data.loc[ind, 'Positives']).sum()
Positive_diff_l = Positive_diff - 2* Positives_CF_sum_std
Positive_diff_u = Positive_diff + 2* Positives_CF_sum_std
Positives_fit = data.loc[ind, 'Tests'] * data.loc[ind, 'odds_hat']/(data.loc[ind, 'odds_hat']+1)
print('All & Counterfactual cases diff {} &  CI ({} {})'.format(Positive_diff, Positive_diff_l, Positive_diff_u))
print('Counterfactual death diff {} CI ({} {})'.format(0.077*Positive_diff, 0.077*Positive_diff_l, 0.077*Positive_diff_u))
print('Analyzed span: {}'.format(max_date-min_date))
```

    New York & Counterfactual cases diff 75648.39608260806 & CI (65411.52499152025 85885.26717369586)
    Connecticut & Counterfactual cases diff 29589.231191984003 & CI (26802.681613687222 32375.780770280784)
    Massachusetts & Counterfactual cases diff 78483.35049817203 & CI (75033.5430372708 81933.15795907326)
    All & Counterfactual cases diff 187234.8756166341 &  CI (175898.072709131 198571.67852413718)
    Counterfactual death diff 14417.085422480824 CI (13544.151598603086 15290.019246358563)
    Analyzed span: 31 days 00:00:00

```python
max_date = min(data.loc[data['State']=='New York', 'Date'].max(), data.loc[data['State']=='Connecticut', 'Date'].max(), data.loc[data['State']=='Massachusetts', 'Date'].max())
min_date = pd.to_datetime('2020-04-14')
for state in  ['New York', 'Connecticut', 'Massachusetts']:
    subdata = data.loc[(data['State']==state) & (data['Date']<=max_date) & (data['Date']>=min_date)]
    Positives_CF = subdata['Tests'] * subdata['Odds CF']/(subdata['Odds CF']+1)
    Positives_CF_std = subdata['Odds CF std'] * subdata['Tests'] /(subdata['Odds CF']+1)**2
    Positives_CF_sum_std = np.sqrt((Positives_CF_std**2).sum())
    Positive_diff = (Positives_CF - subdata['Positives']).sum()
    Positive_diff_l = Positive_diff - 2* Positives_CF_sum_std
    Positive_diff_u = Positive_diff + 2* Positives_CF_sum_std
    Positives_fit = subdata['Tests'] * subdata['odds_hat']/(subdata['odds_hat']+1)
    print('{} & Counterfactual cases diff {} & CI ({} {})'.format(state, Positive_diff, Positive_diff_l, Positive_diff_u))

ind = (data['Date']<=max_date) & (data['Date']>=min_date)
Positives_CF = data.loc[ind, 'Tests'] * data.loc[ind, 'Odds CF']/(data.loc[ind, 'Odds CF']+1)
Positives_CF_std = data.loc[ind, 'Odds CF std'] * data.loc[ind, 'Tests'] /(data.loc[ind, 'Odds CF']+1)**2
Positives_CF_sum_std = np.sqrt((Positives_CF_std**2).sum())
Positive_diff = (Positives_CF - data.loc[ind, 'Positives']).sum()
Positive_diff_l = Positive_diff - 2* Positives_CF_sum_std
Positive_diff_u = Positive_diff + 2* Positives_CF_sum_std
Positives_fit = data.loc[ind, 'Tests'] * data.loc[ind, 'odds_hat']/(data.loc[ind, 'odds_hat']+1)
print('All & Counterfactual cases diff {} &  CI ({} {})'.format(Positive_diff, Positive_diff_l, Positive_diff_u))
print('Counterfactual death diff {} CI ({} {})'.format(0.077*Positive_diff, 0.077*Positive_diff_l, 0.077*Positive_diff_u))
print('Analyzed span: {}'.format(max_date-min_date))
```

    New York & Counterfactual cases diff 75648.39608260806 & CI (65411.52499152025 85885.26717369586)
    Connecticut & Counterfactual cases diff 29589.231191984003 & CI (26802.681613687222 32375.780770280784)
    Massachusetts & Counterfactual cases diff 78483.35049817203 & CI (75033.5430372708 81933.15795907326)
    All & Counterfactual cases diff 187234.8756166341 &  CI (175898.072709131 198571.67852413718)
    Counterfactual death diff 14417.085422480824 CI (13544.151598603086 15290.019246358563)
    Analyzed span: 31 days 00:00:00

## New York City

```python
#max_date = min(data.loc[data['State']=='New York', 'Date'].max(), data.loc[data['State']=='Connecticut', 'Date'].max(), data.loc[data['State']=='Massachusetts', 'Date'].max())
min_date = pd.to_datetime('2020-04-17')
max_date = pd.to_datetime('2020-05-09')
state =  'New York'
subdata = data.loc[(data['State']==state) & (data['Date']<=max_date) & (data['Date']>=min_date)]
Positives_CF = subdata['Tests'] * subdata['Odds CF']/(subdata['Odds CF']+1)
Positives_CF_std = subdata['Odds CF std'] * subdata['Tests'] /(subdata['Odds CF']+1)**2
Positives_CF_sum_std = np.sqrt((Positives_CF_std**2).sum())
Positive_diff = (Positives_CF - subdata['Positives']).sum()
Positive_diff_l = Positive_diff - 2* Positives_CF_sum_std
Positive_diff_u = Positive_diff + 2* Positives_CF_sum_std
Positives_fit = subdata['Tests'] * subdata['odds_hat']/(subdata['odds_hat']+1)
print('{} & Counterfactual cases diff {} & CI ({} {})'.format(state, Positive_diff, Positive_diff_l, Positive_diff_u))

```

    New York & Counterfactual cases diff 54615.59833728758 & CI (46261.90726828152 62969.28940629364)

```python
dsname = 'New York City'

# API information
# https://health.data.ny.gov/Health/New-York-State-Statewide-COVID-19-Testing/xdss-u53e

url = urllib.parse.quote("https://health.data.ny.gov/resource/xdss-u53e.csv/?$limit=5000")
#
dfs = []
offset = 0
while offset >=0:
    url = "https://health.data.ny.gov/resource/xdss-u53e.csv/?$limit=5000&$offset={}".format(offset)
    df = pd.read_csv(url)#, usecols=['test_date', 'total_number_of_tests', 'new_positives']
    dfs.append(df)
    if len(df)==5000:
        offset += 5000
    else:
        offset = -1
dfraw = pd.concat(dfs)
#'test_date=2020-03-15T00:00:00.000'

dfraw = dfraw.rename(columns={'new_positives': 'Positives', 'total_number_of_tests': 'Tests', 'test_date': 'date'})
dfraw['date'] = pd.to_datetime(dfraw['date'])
nyc = dfraw[dfraw.county.isin(['Bronx', 'Kings', 'New York', 'Richmond', 'Queens'])]
#nyc = dfraw[dfraw.county.isin(['Bronx','New York', 'Queens'])]
#counties = (df.groupby('date')['Tests']>0).count()

nys = dfraw.groupby('date').sum()
nys['Odds'] = nys.Positives / (nys.Tests - nys.Positives)
nys['Date'] = pd.to_datetime(nys.index)
nys = nys[nys['Date'] >= '2020-04-17']
# last date of full NYS PAUSE
nys = nys[nys['Date'] <= '2020-05-09']

nyc = nyc.groupby('date').sum()
nyc['Odds'] = nyc.Positives / (nyc.Tests - nyc.Positives)
nyc['Date'] = pd.to_datetime(nyc.index)
nyc = nyc[nyc['Date'] >= '2020-04-17']
# last date of full NYS PAUSE
nyc = nyc[nyc['Date'] <= '2020-05-09']
```

```python
Positive_diff * nyc.Positives.sum()/nys.Positives.sum()
```

    29470.928665071588

```python
Positive_diff_l * nyc.Positives.sum()/nys.Positives.sum()
```

    24963.22315456285

```python
Positive_diff_u* nyc.Positives.sum()/nys.Positives.sum()
```

    33978.63417558033

```python
Positive_diff
```

    54615.59833728758


# Μια **πρακτική**  εισαγωγή στην Αστροστατιστική
---
δηλαδή εφαρμοσμένη στατιστική με ...*εφαρμογές* στην Αστρονομία

### Βιβλιογραφία
1.Statistics, data mining, and machine learning in astronomy: a practical Python guide for the analysis of survey data. (Princeton University Press, 2014).

2.Wall, J. V. & Jenkins, C. R. Practical statistics for astronomers. (Cambridge University Press, 2003).

3.Chattopadhyay, A. K. & Chattopadhyay, T. Statistical Methods for Astronomical Data Analysis. 3, (Springer New York, 2014).

4.Bailer-Jones, Coryn A. L. Practical Bayesian Inference: A Primer for Physical Scientists. Cambridge: Cambridge University Press, 2017. https://doi.org/10.1017/9781108123891.

# Probabillity and Statistics (1)
## 1. Probabillity

Η πιθανότητα είναι ένας τρόπος να ποσοτικοποιούμε τη κατάσταση της γνώσης μας για μια παράμετρο ή υπόθεση.

Probability is a way of quantifying our state of knowledge of a proposition. Examples
of propositions are “this is the spectrum of a star”, “it will rain tomorrow”, “model M
is correct”, and “the period of oscillation lies between 1 and 1.1 seconds”. Our state of
knowledge is inseparable from our belief: the probability reflects what we think about
the proposition, based on information, data, etc., that we have. Other people with other
information may well assign a different probability.
Let A and B be two propositions which have probabilities P (A) and P (B) respectively.
Without being too formal, the following are the essential axioms of probability. 1 Probabilities are real numbers that satisfy

$$ 0 \leq P(A) \leq 1$$

Denote as A the complement of A (i.e. A doesn’t happen or isn’t true). Then

$$ P(A') = 1 - P(A) $$

If two propositions A and B are mutually exclusive then

$$ P(A \text{ or }B)= P(A) +P(B) $$

The conditional probability of A occurring given that B is true (e.g. A = “rain”,
B = “cloudy”) is written $ P (A | B) $. The joint probability that both are true is written
$P (A, B)$. It follows logically that we can factorize this and write it as
$$ P (A \text{ and } B) ≡ P (A, B) = P (A| B)P (B) = P (B | A)P (A)$$

If A and B are mutually exclusive, $P (A, B) = 0$. If A and B are independent, $P (B | A) =
P (B) \text{ and } P (A| B) = P (A)$, in which case
$$P (A, B) = P (A)P (B) $$
Equation 1.5 is simple, yet it is one of the most important for inference. It can be rear-
ranged and written as
$$
P (A | B) = \frac{P (A, B)}{P (B)}
$$
Note that $P (A | B) \neq P (B | A) $.

If you’re not convinced, take an example: think of A = “being female”, B = “being pregnant”. Suppose that the average human female gives birth to two children in her life (gestation period $9/12$ of a year) and the life expectancy is 80 years, then for a female chosen at random, $P (B | A) \simeq 2 \times (9/12)/80 \simeq 0.02$. Different assumptions will give a slightly different result, but unless you work in a maternity ward it will be small. Yet I think all will agree that P (A | B) = 1, for all common definitions of “female” and “pregnant”.
In general it must hold that
$$ P (A) = P (A | B)P (B) + P (A | B')P (B') $$
because B is either true or is not true, so all possibilities are covered. More generally, if
$ \{B_i \}$ is the set of all possible propositions, i.e. $\sum _i P (B_i ) = 1$, then it follows that
$$
P (A) =\sum _i P (A, B_i ) =\sum _i P (A | B_i )P (B_i )
$$
This is called the marginal probability of A, where “marginal” just means “average”, here
over the $\{B_i \}$.

### Bayes’ theorem and the principle of inference
If we rearrange the equation for joint probabilities, we get
$$
P (B | A) = \frac{P (A | B)P (B)}{P (A)}
$$
This is called Bayes’ theorem. It was once referred to as inverse probability, because it
relates the probability of B given A to the probability of A given B. It is fundamental in
inference for the following reason. Taking M as a model (or hypothesis) and D as some
data that we have obtained to test the model, then we can write Bayes’ theorem as
$$
P (M | D) = \frac{P (D | M )P (M )}{P (D)}
$$
When we set up an experiment we can normally determine $P (D | M )$, the probability of
observing the data (or perhaps a quantity derived from the data) under the assumption
that the model is true. For example, we might be looking for the presence of spectral line
(D = line is present) under the assumption that it is produced by a particular trace gas
in an exoplanet’s atmosphere. We then set up a model M for the atmosphere and our
measurements, and compute the probability $P (D | M )$ that this would allow us to observethe line. Note that this is not the probability that the model is true, given the data. That
quantity is $P (M | D)$ (recall the example above about pregnancy and females). Observing
the line does not necessarily indicate the presence of the gas, because there may be other
origins of the line (i.e. other models).
To infer $P (M | D)$ we use Bayes’ theorem. Equation 1.14 tells us that we need two other
quantities in addition to $P (D | M )$. The first is $P (M )$, the unconditional probability that
the model M is true. This is called the prior probability, prior here meaning “prior to using
the data”. The other quantity we need is $P (D)$, the unconditional probability of the data,
meaning its probability independent of any particular model. Let $\{M_i \}$ be the complete set
of all models (mutually exclusive and exhaustive). Using equation 1.9 we can write
$$
P (D) =\sum _i P (D | M_i )P (M_i )
$$
We now have all terms needed to compute $P (M | D)$, the posterior probability that M is
true. This differs from the prior probability in that it is conditional on the data. Bayes’
theorem underpins much of inference, so I will have a lot more to say about what it is and
how we use it. We will return to it again in chapter 3.

### Discrete and continuous probability distributions
So far we have considered discrete variables, whereby $P (B_i )$ gives the actual probability
of proposition $B_i$ being true or of event $B_i$ occurring. We sometimes refer to the set of
probabilities over all events as the probability mass function (PMF).
For a continuous variable x we instead deal with the probability density function (PDF)
$P (x)$. This is a density – probability per unit x – not a probability. $P (x)dx$ is the in-
finitesimal probability of x in the range x to $x + dx$, so a finite probability is obtained by
integrating over a region. Specifically, the probability of x lying between $x_1$ and $x_2$ is
$$
\int _{x_1} ^{x_2} P (x) dx
$$
Probability is dimensionless: it has no units. Therefore the density $P (x)$ has units $1/x$.
Note that $P (x | y)$ is a PDF in x, not in y. The thing after the bar just gives conditional
information. The variables x and y need not have the same units, which is another reason
why $P (x | y) \neq P (y | x)$ in general. I will use the same symbol – upper case P – to refer to
both PMFs and PDFs. It will be clear from the context which I mean.
The range of the variable x over which the PDF (or PMF) is non-zero is called the
support of the distribution. If it is non-zero over all real values, the distribution is said to
have infinite support. If it has a bound, e.g. it is non-zero only for positive values, it is
referred to as having semi-infinite support.1.2.6 Normalization
A bona fide PDF is always normalized, i.e.
$$
\int P (x) dx = 1
$$
and likewise
$$
\sum _i P (x i ) = 1
$$

for discrete distributions. Distributions that cannot be normalized are called improper. An
example is the uniform distribution with infinite support (section 1.4.4). It is not normaliz-
able because its integral is infinite. We will nonetheless see that improper distributions can
be useful in certain contexts.
We will sometimes work with unnormalized distributions, which I will indicate
using an asterisk $P^*$ . For an unnormalized PDF $P^* (x)$ the normalization constant is $\int P (x) dx$, which is independent of x (because we’ve integrated over it). The normalized PDF is therefore
$$
P(x) = \frac{P^* (x)}{\int P^*(x) dx}
$$
and the normalized discrete distribution (probability mass function) is
$$
P(x_i )=\frac{P ^*(x_i )}{\sum _i P (x_i )}
$$

### RANDOM VARIABLES FROM [1]

## Τυχαίες μεταβλητές
Μια τυχαία (ή στοχαστική) μεταβλητή είναι μια μεταβλήτη η οποία προέρχεται από μια ποσότητα που υπόκεινται σε τυχαίες διακυμάνσης. Προέρχεται δηλαδή από μια στατιστική κατανομή. Υπαρχουν δύο ειδών τυχαίες μεταβλήτες, οι διακριτές (που μπορεί να προέρχονται για παράδειγμα από μια διωνυμική ή poisson κατανομή) και οι συνεχείς (gaussian αντιστοιχα)

Δύο τυχαίες μεταβλητές $x,y$ είναι ανεξάρτητες να και μόνο αν:
$$p(x,y)=p(x)p(y)$$
Στη περίπτωση όπου δύο τυχαίες μεταβλητές δεν είναι ανεξάρτητες τότε:
$$ p(x,y)=p(x|y)p(y)=p(y|x)p(x) $$
Η marginal πιθανότητα δίνεται από
$$
p(x)= \int p(x,y)dy = \int p(x|y)p(y) dy
$$
Στο παρακάτω σχήμα γίνεται εμφανές η πραγματική κατανομή δυο μή ανεξαρτήτων μεταβλητών. Από τη συνδυασμένη κατανομή παίρνουμε slices τα οποία φαίνονται στα δεξιά.

Δεν αρκεί όμως απλά να πάρουμε το slice της κατανομής. Η $p(x|1.5)$ είναι και αυτή μια κατανομή, άρα θα πρέπει συνολικά να έχει πιθανότητα 1. Άρα πρέπει να κανονικοποιήσουμε τη κατανομή, δηλαδή να διαιρέσουμε με τη marginal κατανομή της $y$ στο συγκεκριμένο slice, $$p(x|1.5)=\frac{p(x,y=1.5)}{p(y=1.5)}$$

Ο κανόνας του Bayes χτίζεται γενικεύοντας τις παραπαπάνω σχέσεις:
$$
p(y|x)=\frac{p(x|y)p(y)}{p(x)}=\frac{p(x|y)p(y)}{\int p(x|y)p(y)dy}
$$

Ο απλός αυτός κανόνας, μια απλη αγλεβρική αναδιάταξη δηλαδή, μας ανοίξε νέους ορίζοντες στο να κάνουμε στατιστική. Ας δούμε ένα απλό παράδειγμα.

Έστω ότι παρατηρούμε κάποιον -τον φώτη- να φοράει κασκόλ της αεκ, ποια η πιθανότητα να είναι οπαδός της αεκ?
Τα ενδεχόμενα είναι δύο
* ενδεχόμενο Α ο φώτης είναι αεκ
* ενδεχόμενο Α' ο φώτης δεν είναι αεκ

Αναζητούμε τη πιθανότητα δεδομένου ο φώτης να φοράει κασκολ της αεκ, να είναι και αεκ. Δηλαδή τη πιθανότητα $P(\text{αεκ}|\text{κασκολ αεκ})$.

Χρησιμοποιώντας το κανόνα του Bayes έχουμε:

$$
P(\text{αεκ}|\text{κασκολ αεκ})=
\frac{P(\text{αεκ}) P(\text{κασκολ αεκ}|\text{αεκ})}{P(\text{αεκ}) P(\text{κασκολ αεκ}|\text{αεκ})+
P(\text{οχι αεκ}) P(\text{κασκολ αεκ}|\text{οχι αεκ})}
$$

Ας μελετήσουμε τη σχέση αυτή βήμα - βήμα:

Ο όρος $P(\text{αεκ})$ αποτελεί τη πιθανότητα να είναι κάποιος ΑΕΚ γενικά. Αυτη η πιθανότητα ονομάζεται **prior** και είναι η καινοτομία (καλή ή και κακή) που μας φέρνει η bayesian προσέγγιση. Θα πρέπει λοιπόν να έχουμε μια ένδειξη της υπόθεσης μας. Θεωρώντας ότι η ΑΕΚ είναι μια γνωστή ομάδα, οπού κρατάει ένα ποσοστό κοντά αλλα χαμηλότερα στο $1/3$ στην αθήνα θεωρούμε μια τιμή $1/5$.

Ο όρος $P(\text{κασκολ ΑΕΚ}|\text{ΑΕΚ})$ αποτελεί τη πιθανότητα του να μην είναι τύχαιο κάποιος να φοράει ένα κασκολ της ΑΕΚ. Πρόκειται για τη σχέση πιθανοφάνειας ή **likehood**. Θεωρώντας ότι είναι αρκετά σπάνιο φαινόμενο αυτό δίνουμε μια τιμή $0.9$. Σημειώστε ότι χωρίς τη χρήση της bayesian προσέγγισης μάλλον η εκτίμηση μας θα ήταν αντίστοιχη αυτού του αριθμού.

Ο όρος στον παρονομάστη είναι ουσιάστικα ένας όρος κανονικοποιήσης. Στα δικά μας δεδομένα είναι έυκολο να υπολογιστεί καθώς έχουμε μόνο δύο ενδεχώμενα. Σε πραγματικά δεδομένα -όπου θέλουμε σαν αποτέλεσμα μια κατανομή- όμως είναι πολύπλοκο να υπολογιστεί. Αυτό δεν μας πειράζει όμως καθώς μπορούμε έυκολα να κάνουμε τη κανονικοποίηση χωρίς να τον υπολογίσουμε.

\begin{align}
P(\text{αεκ}) = P_0 = 0.2 \\
P(\text{όχι αεκ}) = 1-P_0 = 0.8 \\
P(\text{κασκολ αεκ}|\text{αεκ}) = 0.9 \\
P(\text{κασκολ αεκ}|\text{οχι αεκ}) = 0.1 \\
\end{align}

Η τελική πιθανότητα, ονομάζεται **posterior** υπολογίζεται λοιπόν περίπου $70\%$.

### 1.1 Expectation

We often want to summarize probability distributions with just a few numbers. Two sum-
mary statistics of particular interest are the mean, which indicates the location of the dis-
tribution, and the standard deviation, which is a measure of its width. Many of the most
commonly occurring distributions are fully defined by these two parameters.
The expectation value or mean of a continuous variable x with a PDF P (x) is defined as
$$
E[x] =\int x P (x) dx
$$
For a discrete variable with PMF $P (x_i )$ it is defined as
$$
E[x] = \sum _i x_i P (x_i )
$$
The expectation value is sometimes written as $\mu$. The expectation value of a function $f (x)$
is defined as
$$
E[f (x)] = \int f (x) P (x) dx
$$
If we have a set of N data points drawn from a continuous distribution, then we can use
equation 1.22 as an approximation for equation 1.21. Each data point occurs once, so im-
plicitly $P (x_i ) = 1/N$ , and
$$
E[x] \simeq \frac{1}{N} \sum _i ^N x_i = \bar{x}
$$
which is called the sample mean. The law of large numbers says that in the limit $N \to ]inf$ the sample mean tends towards the (true) mean.

### 1.2 Variance
The variance of a random variable x with expectation value $E[x]$ is defined as
$$
\text{Var} (x) = \int (x-E[x])^2 P(x) dx = \dots = E[x^2]-E[x]^2
$$
We can remember this as “the variance is the expectation of the square minus the square of
the expectation” (or just “mean square minus square mean”). If we know the true mean μ,
then for a set of N data points $\{x_i \}$, the variance can be approximated by
$$
\text{Var}(x)\simeq \frac{1}{N} \sum _i ^N (x_i -\mu)^2
$$
This is an approximation because of the finite sample size. If we don’t know $\mu$ then we
might want to estimate the variance as
$$
V_x= \frac{1}{N} \sum _i ^N (x_i -\bar{x})^2=\bar{x^2}-\bar{x}^2
$$
However, if we calculate the expected value of this we find

**[page 82 in 1]**
$$
E[V_x]=\frac{N-1}{N}\text{Var}(x)
$$
which is not equal to the true variance for finite N . It is therefore a biased estimate; specif-
ically it is an underestimate (see section 2.1 for a discussion of estimator bias). The origin
of this bias is that $V_x$ uses the estimated (sample) mean of the data (equation 1.24), whereas
the definition of variance uses the true mean. As we have computed this bias we can correct
for it, to give the sample variance, which is defined as
$$
\hat{\text{Var}(x)}=\frac{1}{N-1}\sum _i ^N (x_i -\bar{x})^2
$$
which is an unbiased estimate of the true variance.

The standard deviation is defined as
$$
\sigma =\sqrt{\text{Var}(x)}
$$
which for a set of data of known mean is
$$
\sigma\simeq \sqrt{ \frac{1}{N} \sum _i ^N (x_i -\mu)^2}
$$
where the approximation symbol has again been used because we have a finite-sized sam-
ple. The sample standard deviation is the square root of the sample variance
$$
\hat{\sigma}=\sqrt{\frac{1}{N-1}\sum _i ^N (x_i -\bar{x})^2}
$$

```python
from scipy import stats
from scipy import integrate
import numpy as np
%matplotlib inline
import matplotlib.pyplot as plt
from matplotlib import rc,rcParams
rc('text', usetex=True)
rcParams['figure.figsize'] = (9, 5.5)
rcParams['ytick.labelsize'],rcParams['xtick.labelsize'] = 17.,17.
rcParams['axes.labelsize']=19.
rcParams['legend.fontsize']=17.
rcParams['axes.titlesize']=20.
import seaborn as sb
sb.despine()
sb.set_style('white', {'xes.linewidth': 0.5, 'axes.edgecolor':'black'})
sb.despine(left=True)
```

    <Figure size 648x396 with 0 Axes>

## Uniform

```python
u=stats.uniform
xx=np.linspace(0,1)
```

```python
x=u.rvs(size=10)
```

```python
plt.plot(xx,u.pdf(xx))
```

```python
plt.plot(xx,u.cdf(xx))
```

```python
du=stats.randint
```

```python
xxd=np.linspace(0,99,10,dtype=int)
```

```python
np.diff(xxd)[0]
```

    11

```python
plt.bar(xxd,du.pmf(xxd,low=0,high=1000),width=np.diff(xxd)[0])
```

```python
xd=du.rvs(size=100,low=1,high=100)
```

```python
plt.hist(xd,density=True,bins=20)
```

```python
plt.hist(xd**2,density=True,bins=20)
```

```python
def E(x,p):
    return np.sum(p*x)
def Var(x,p):
    return E(x**2,p)-E(x,p)**2
```

```python
N=xd.shape[0]
```

```python
pxd=(np.ones(shape=N)/N)
```

```python
((xd**2).mean()-xd.mean()**2)
```

    936.3019000000004

```python
Ε()
```

```python
np.sum(xd**2-xd.mean()**2)/(N-1)
```

    945.7594949494953

```python
E(xd,pxd),Var(xd,pxd)
```

    (50.91000000000001, 936.3018999999986)

## 1.3 Distributions

```python
Binomial
This probability distribution describes processes in which an event can have only one of
two possible outcomes. Examples include tossing a coin, detecting something at a security
check, or winning the lottery. Let p be the probability of one event, call it “success”; 1 − p
is the probability of the other event (“failure”). If the trial is repeated independently n
times, then we are interested in the probability of getting exactly r successes, which we
can label P (r | p, n). Suppose that the first r trials are successes and the remainder, n − r,
are all failures. As the trials are independent, the probability of this sequence is just the
product of the probabilities, p r (1 − p) n−r . This is the probability of just one particular
sequence of trials. The number of unique sequences with this probability is the number of
ways of selecting r from n (without replacement), which is n C r (see section 1.7). As these
sequences are mutually exclusive we just sum the probability n C r times. This gives us thebinomial distribution
```


# Quenching and drifting examples
In this notebook we apply the quenching and drifting stages to a track dataset

```python
# This is need so you can import larndsim without doing python setup.py install
import os,sys,inspect
currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
parentdir = os.path.dirname(currentdir)
sys.path.insert(0,parentdir)
```

```python
from larndsim import consts
consts.load_detector_properties("../larndsim/detector_properties/module0.yaml",
                                "../larndsim/pixel_layouts/multi_tile_layout-2.2.16.yaml")

from larndsim import quenching, drifting
from larndsim import quenching_ep, drifting_ep
import importlib
importlib.reload(drifting)
importlib.reload(quenching)
importlib.reload(drifting_ep)
importlib.reload(quenching_ep)

from math import ceil

import matplotlib.pyplot as plt

import numpy as np
import h5py

from numpy.lib import recfunctions as rfn
import torch

def torch_from_structured(tracks):
    tracks_np = rfn.structured_to_unstructured(tracks, copy=True, dtype=np.float32)
    return torch.from_numpy(tracks_np).float()

def structered_from_torch(tracks_torch, dtype):
    return rfn.unstructured_to_structured(tracks_torch.cpu().numpy(), dtype=dtype)
```

```python
with h5py.File('edepsim_1M.h5', 'r') as f:
    tracks = np.array(f['segments'])

y_start = np.copy(tracks['y_start'] )
y_end = np.copy(tracks['y_end'])
y = np.copy(tracks['y'])

tracks['y_start'] = np.copy(tracks['z_start'])
tracks['y_end'] = np.copy(tracks['z_end'])
tracks['y'] = np.copy(tracks['z'])

tracks['z_start'] = y_start
tracks['z_end'] = y_end
tracks['z'] = y

# tracks = tracks[:1000]
```

```python
tracks_box = np.copy(tracks)
tracks_birks = np.copy(tracks)
print(tracks_box.shape)
print(tracks_birks.shape)
```

    (154359,)
    (154359,)

```python
threadsperblock = 256
blockspergrid = ceil(tracks.shape[0] / threadsperblock)

quenching.quench[blockspergrid,threadsperblock](tracks_box, consts.box)
quenching.quench[blockspergrid,threadsperblock](tracks_birks, consts.birks)
```

## Recombination

The number of ionized electrons must be corrected by the recombination effect. Two models have been implemented:

- Birks (Amoruso et al. NIM A 523 (2004) 275)
\begin{equation}
R_{Birks} = \frac{A_b}{1+k_b/\epsilon\cdot dE/dx }
\end{equation}

- Modified box (Baller, 2013 JINST 8 P08005)
\begin{equation}
R_{box} = \frac{\log(\alpha + \beta \frac{dE/dx}{E\rho})}{\beta/\epsilon\cdot dE/dx}
\end{equation}

where $\epsilon=E\rho_{\mathrm{LAr}}$.

```python
%matplotlib notebook

def plot_tracks(tracks_box, tracks_birks):
    fig, ax = plt.subplots(1, 1)

    recomb_birks = tracks_birks["n_electrons"] / (consts.MeVToElectrons * tracks_birks["dE"])
    recomb_box = tracks_box["n_electrons"] / (consts.MeVToElectrons * tracks_box["dE"])

    ax.plot(np.sort(tracks_box["dEdx"]), recomb_box[np.argsort(tracks_box["dEdx"])],
            label="Box model", lw=2)
    ax.plot(np.sort(tracks_birks["dEdx"]), recomb_birks[np.argsort(tracks_birks["dEdx"])],
            label="Birks model", lw=2)
    ax.set_xlabel("$dE/dx$ [MeV/cm]")
    ax.set_ylabel("Recombination factor")
    ax.set_xlim(0,10)
    _ = ax.legend()

plot_tracks(tracks_box, tracks_birks)

#from structured to unstructerd
tracks_box_torch = torch_from_structured(np.copy(tracks))
tracks_birks_torch = torch_from_structured(np.copy(tracks))
quenching_ep.quench(tracks_box_torch, consts.box, tracks.dtype.names)
quenching_ep.quench(tracks_birks_torch, consts.birks, tracks.dtype.names)
plot_tracks(structered_from_torch(tracks_box_torch, tracks.dtype),
            structered_from_torch(tracks_birks_torch, tracks.dtype))
```

    <IPython.core.display.Javascript object>

    <IPython.core.display.Javascript object>

## Electron lifetime
The electrons traveling towards the anode are absorbed by impurities present in the liquid argon. The decrease follows an exponential curve $N_e^{\mathrm{anode}}/N_e=\exp(-t_{\mathrm{drift}}/e_{\mathrm{lifetime}})$.

```python
def plotDriftTime(tracks):
    fig, ax = plt.subplots(1, 1)
    drift_time = abs(tracks["z"] - consts.tpc_borders[0][2][0])/consts.vdrift
    ax.plot(np.sort(tracks["z"]), np.exp(-drift_time / consts.lifetime)[np.argsort(tracks['z'])], lw=2)
    ax.set_ylabel("$N_{e}^{\mathrm{anode}}/N_{e}$")
    _ = ax.set_xlabel("$z$ coordinate [cm]")

tracks_quenched = np.copy(tracks)
quenching.quench[blockspergrid,threadsperblock](tracks_quenched, consts.box)
tracks_drifted = np.copy(tracks_quenched)
drifting.drift[blockspergrid,threadsperblock](tracks_drifted)
plotDriftTime(tracks_drifted)

tracks_quenched_torch = torch_from_structured(np.copy(tracks))
quenching_ep.quench(tracks_quenched_torch, consts.box, tracks.dtype.names)
tracks_quenched_ep = structered_from_torch(tracks_quenched_torch, tracks.dtype)
tracks_drifted_torch = torch_from_structured(np.copy(tracks_quenched_ep))
drifting_ep.drift(tracks_drifted_torch, tracks.dtype.names)
tracks_drifted_ep = structered_from_torch(tracks_drifted_torch, tracks.dtype)
plotDriftTime(tracks_drifted_ep)
```

    <IPython.core.display.Javascript object>

    <IPython.core.display.Javascript object>

## Diffusion
The diffusion coefficients (longitudinal and transverse) are proportional to $\sqrt{2t_{\mathrm{drift}}}$ (which in turn is given by $(z_{\mathrm{start}} - z_{\mathrm{anode}})/v_{\mathrm{drift}}$).

```python
def plotDiffusion(tracks):
    fig, ax = plt.subplots(1, 1)
    ax.plot(np.sort(tracks["z"]), tracks["long_diff"][np.argsort(tracks["z"])], label="Longitudinal diffusion", lw=2)
    ax.plot(np.sort(tracks["z"]), tracks["tran_diff"][np.argsort(tracks["z"])], label="Transverse diffusion", lw=2)
    ax.set_xlabel("$z$ start coordinate [cm]")
    ax.set_ylabel("Diffusion coefficient")
    _ = ax.legend()

plotDiffusion(tracks_drifted)
plotDiffusion(tracks_drifted_ep)
```

    <IPython.core.display.Javascript object>

    <IPython.core.display.Javascript object>


```
pip install strawberryfields
```

    Requirement already satisfied: strawberryfields in /usr/local/lib/python3.6/dist-packages (0.14.0)
    Requirement already satisfied: python-dateutil>=2.8.0 in /usr/local/lib/python3.6/dist-packages (from strawberryfields) (2.8.1)
    Requirement already satisfied: appdirs in /usr/local/lib/python3.6/dist-packages (from strawberryfields) (1.4.4)
    Requirement already satisfied: scipy>=1.0.0 in /usr/local/lib/python3.6/dist-packages (from strawberryfields) (1.4.1)
    Requirement already satisfied: networkx>=2.0 in /usr/local/lib/python3.6/dist-packages (from strawberryfields) (2.4)
    Requirement already satisfied: urllib3>=1.25.3 in /usr/local/lib/python3.6/dist-packages (from strawberryfields) (1.25.9)
    Requirement already satisfied: numpy>=1.17.4 in /usr/local/lib/python3.6/dist-packages (from strawberryfields) (1.18.5)
    Requirement already satisfied: numba in /usr/local/lib/python3.6/dist-packages (from strawberryfields) (0.48.0)
    Requirement already satisfied: sympy>=1.5 in /usr/local/lib/python3.6/dist-packages (from strawberryfields) (1.6.1)
    Requirement already satisfied: thewalrus>=0.12 in /usr/local/lib/python3.6/dist-packages (from strawberryfields) (0.12.0)
    Requirement already satisfied: quantum-blackbird>=0.2.3 in /usr/local/lib/python3.6/dist-packages (from strawberryfields) (0.2.4)
    Requirement already satisfied: toml in /usr/local/lib/python3.6/dist-packages (from strawberryfields) (0.10.1)
    Requirement already satisfied: requests>=2.22.0 in /usr/local/lib/python3.6/dist-packages (from strawberryfields) (2.23.0)
    Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.6/dist-packages (from python-dateutil>=2.8.0->strawberryfields) (1.12.0)
    Requirement already satisfied: decorator>=4.3.0 in /usr/local/lib/python3.6/dist-packages (from networkx>=2.0->strawberryfields) (4.4.2)
    Requirement already satisfied: llvmlite<0.32.0,>=0.31.0dev0 in /usr/local/lib/python3.6/dist-packages (from numba->strawberryfields) (0.31.0)
    Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from numba->strawberryfields) (49.1.0)
    Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.6/dist-packages (from sympy>=1.5->strawberryfields) (1.1.0)
    Requirement already satisfied: dask[delayed] in /usr/local/lib/python3.6/dist-packages (from thewalrus>=0.12->strawberryfields) (2.12.0)
    Requirement already satisfied: antlr4-python3-runtime==4.7.2 in /usr/local/lib/python3.6/dist-packages (from quantum-blackbird>=0.2.3->strawberryfields) (4.7.2)
    Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests>=2.22.0->strawberryfields) (2.10)
    Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests>=2.22.0->strawberryfields) (2020.6.20)
    Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests>=2.22.0->strawberryfields) (3.0.4)
    Requirement already satisfied: cloudpickle>=0.2.1; extra == "delayed" in /usr/local/lib/python3.6/dist-packages (from dask[delayed]->thewalrus>=0.12->strawberryfields) (1.3.0)
    Requirement already satisfied: toolz>=0.7.3; extra == "delayed" in /usr/local/lib/python3.6/dist-packages (from dask[delayed]->thewalrus>=0.12->strawberryfields) (0.10.0)

```
import strawberryfields as sf
from strawberryfields import ops
import numpy as np
```

```
prog = sf.Program(2)
with prog.context as q:
  ops.Fock(2)|q[0]
  ops.Fock(1)|q[1]
  ops.BSgate(0.2, 0.1)|q
  ops.MeasureFock()|q

eng = sf.Engine("fock", backend_options={"cutoff_dim":5})
result = eng.run(prog)
print(np.sum(result.samples))
```

    3

```
prog1 = sf.Program(8)
with prog1.context as q:
  ops.S2gate(1.0) | (q[0], q[4])
  ops.S2gate(1.0) | (q[1], q[5])
  ops.S2gate(1.0) | (q[3], q[7])
  ops.MeasureFock()|q

eng = sf.Engine("gaussian")
result = eng.run(prog1, shots=5)
print(result.samples)
```

    /usr/local/lib/python3.6/dist-packages/strawberryfields/backends/gaussianbackend/backend.py:232: UserWarning: Cannot simulate non-Gaussian states. Conditional state after Fock measurement has not been updated.

    [[0 0 0 2 0 0 0 2]
     [0 0 0 2 0 0 0 2]
     [2 3 0 2 2 3 0 2]
     [2 0 0 1 2 0 0 1]
     [0 0 0 0 0 0 0 0]]

```
sf.store_account("qUOm1ptO039jCzVTTEUSd9SHZwkAs5dI8Rj8eIAZ")
sf.ping()
```

    You have successfully authenticated to the platform!

```
eng=sf.RemoteEngine("X8")
result = eng.run(prog1, shots=5)
mes = result.samples
print(mes)
```

    2020-07-15 15:58:09,153 - INFO - Job 18d4495c-da5a-4551-9c91-2a011f89d8cb was successfully submitted.
    2020-07-15 15:58:35,194 - INFO - The remote job 18d4495c-da5a-4551-9c91-2a011f89d8cb has been completed.

    [[0 0 0 0 1 0 0 0]
     [0 0 0 0 0 1 0 1]
     [0 0 0 0 1 1 0 0]
     [0 0 0 1 0 0 0 0]
     [0 0 0 0 0 0 0 0]]

```
from strawberryfields.utils import random_interferometer

prog = sf.Program(8, name="remote_job")
U = random_interferometer(4)

with prog.context as q:
    # Initial squeezed states
    # Allowed values are r=1.0 or r=0.0
    ops.S2gate(1.0) | (q[0], q[4])
    ops.S2gate(1.0) | (q[1], q[5])
    ops.S2gate(1.0) | (q[3], q[7])

    # Interferometer on the signal modes (0-3)
    ops.Interferometer(U) | (q[0], q[1], q[2], q[3])
    ops.BSgate(0.543, 0.123) | (q[2], q[0])
    ops.Rgate(0.453) | q[1]
    ops.MZgate(0.65, -0.54) | (q[2], q[3])

    # *Same* interferometer on the idler modes (4-7)
    ops.Interferometer(U) | (q[4], q[5], q[6], q[7])
    ops.BSgate(0.543, 0.123) | (q[6], q[4])
    ops.Rgate(0.453) | q[5]
    ops.MZgate(0.65, -0.54) | (q[6], q[7])

    ops.MeasureFock() | q

eng = sf.RemoteEngine("X8")
results = eng.run(prog, shots=1000)
print(results)

```

```

```