import os

import gc

import sys

import random

import logging

import datetime

import numpy as np

import pandas as pd

import seaborn as sns

import lightgbm as lgb

from plotly import tools

from pathlib import Path

import plotly.graph_objs as go

import matplotlib.pyplot as plt

import plotly.figure_factory as ff

from sklearn.metrics import mean_squared_error

from sklearn.model_selection import StratifiedKFold

from plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot

init_notebook_mode(connected=True)
IS_LOCAL = False

if(IS_LOCAL):

    PATH="../input/elo/"

else:

    PATH="../input/"

os.listdir(PATH)
train_df=pd.read_csv(PATH+'train.csv')

test_df=pd.read_csv(PATH+'test.csv')

historical_trans_df=pd.read_csv(PATH+'historical_transactions.csv')

new_merchant_trans_df=pd.read_csv(PATH+'new_merchant_transactions.csv')

merchant_df=pd.read_csv(PATH+'merchants.csv')
print("Train: rows:{} cols:{}".format(train_df.shape[0], train_df.shape[1]))

print("Test:  rows:{} cols:{}".format(test_df.shape[0], test_df.shape[1]))

print("Historical trans: rows:{} cols:{}".format(historical_trans_df.shape[0], historical_trans_df.shape[1]))

print("New merchant trans:  rows:{} cols:{}".format(new_merchant_trans_df.shape[0], new_merchant_trans_df.shape[1]))

print("Merchants: rows:{} cols:{}".format(merchant_df.shape[0], merchant_df.shape[1]))
train_df.sample(3).head()
test_df.sample(3).head()
historical_trans_df.sample(3).head()
new_merchant_trans_df.sample(3).head()
merchant_df.sample(3).head()
def missing_data(data):

    total = data.isnull().sum().sort_values(ascending = False)

    percent = (data.isnull().sum()/data.isnull().count()*100).sort_values(ascending = False)

    return pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
missing_data(train_df)
missing_data(test_df)
missing_data(historical_trans_df)
missing_data(new_merchant_trans_df)
missing_data(merchant_df)
def get_categories(data, val):

    tmp = data[val].value_counts()

    return pd.DataFrame(data={'Number': tmp.values}, index=tmp.index).reset_index()
def get_target_categories(data, val):

    tmp = data.groupby('target')[val].value_counts()

    return pd.DataFrame(data={'Number': tmp.values}, index=tmp.index).reset_index()
def draw_trace_bar(data_df,color='Blue'):

    trace = go.Bar(

            x = data_df['index'],

            y = data_df['Number'],

            marker=dict(color=color),

            text=data_df['index']

        )

    return trace

def draw_trace_histogram(data_df,target,color='Blue'):

    trace = go.Histogram(

            y = data_df[target],

            marker=dict(color=color)

        )

    return trace
def plot_bar(data_df, title, xlab, ylab,color='Blue'):

    trace = draw_trace_bar(data_df, color)

    data = [trace]

    layout = dict(title = title,

              xaxis = dict(title = xlab, showticklabels=True, tickangle=0,

                          tickfont=dict(

                            size=10,

                            color='black'),),

              yaxis = dict(title = ylab),

              hovermode = 'closest'

             )

    fig = dict(data = data, layout = layout)

    iplot(fig, filename='draw_trace')
def plot_two_bar(data_df1, data_df2, title1, title2, xlab, ylab):

    trace1 = draw_trace_bar(data_df1, color='Blue')

    trace2 = draw_trace_bar(data_df2, color='Lightblue')

    fig = tools.make_subplots(rows=1,cols=2, subplot_titles=(title1,title2))

    fig.append_trace(trace1,1,1)

    fig.append_trace(trace2,1,2)

    fig['layout']['xaxis'].update(title = xlab)

    fig['layout']['xaxis2'].update(title = xlab)

    fig['layout']['yaxis'].update(title = ylab)

    fig['layout']['yaxis2'].update(title = ylab)

    fig['layout'].update(showlegend=False)

    iplot(fig, filename='draw_trace')
def plot_target_distribution(var):

    hist_data = []

    varall = list(train_df.groupby([var])[var].nunique().index)

    for i, varcrt in enumerate(varall):

        classcrt = train_df[train_df[var] == varcrt]['target']

        hist_data.append(classcrt)

    fig = ff.create_distplot(hist_data, varall, show_hist=False, show_rug=False)

    fig['layout'].update(title='Target variable density plot group by {}'.format(var), xaxis=dict(title='Target'))

    iplot(fig, filename='dist_only')
plot_two_bar(get_categories(train_df,'feature_1'), get_categories(test_df,'feature_1'),

             'Train data', 'Test data',

             'Feature 1', 'Number of records')
plot_target_distribution('feature_1')
plot_two_bar(get_categories(train_df,'feature_2'), get_categories(test_df,'feature_2'),

             'Train data', 'Test data',

             'Feature 2', 'Number of records')
plot_target_distribution('feature_2')
plot_two_bar(get_categories(train_df,'feature_3'), get_categories(test_df,'feature_3'),

             'Train data', 'Test data',

             'Feature 3', 'Number of records')
plot_target_distribution('feature_3')
plot_two_bar(get_categories(train_df,'first_active_month'), get_categories(test_df,'first_active_month'),

             'Train data', 'Test data',

             'First active month', 'Number of records')
plot_bar(get_categories(historical_trans_df,'category_1'),

             'Category 1 distribution', 'Category 1', 'Number of records')
plot_bar(get_categories(historical_trans_df,'category_2'),

             'Category 2 distribution', 'Category 2', 'Number of records','red')
plot_bar(get_categories(historical_trans_df,'category_3'),

             'Category 3 distribution', 'Category 3', 'Number of records','magenta')
plot_bar(get_categories(historical_trans_df,'city_id'),

             'City ID distribution', 'City ID', 'Number of records','lightblue')
plot_bar(get_categories(historical_trans_df,'merchant_category_id'),

             'Merchant Cateogory ID distribution', 'Merchant Category ID', 'Number of records','lightgreen')
plot_bar(get_categories(historical_trans_df,'state_id'),

             'State ID distribution', 'State ID', 'Number of records','brown')
plot_bar(get_categories(historical_trans_df,'subsector_id'),

             'Subsector ID distribution', 'Subsector ID', 'Number of records','orange')
historical_trans_df['purchase_date'] = pd.to_datetime(historical_trans_df['purchase_date'])

historical_trans_df['month'] = historical_trans_df['purchase_date'].dt.month

historical_trans_df['dayofweek'] = historical_trans_df['purchase_date'].dt.dayofweek

historical_trans_df['weekofyear'] = historical_trans_df['purchase_date'].dt.weekofyear
def plot_scatter_data(data, xtitle, ytitle, title, color='blue'):

    trace = go.Scatter(

        x = data.index,

        y = data.values,

        name=ytitle,

        marker=dict(

            color=color,

        ),

        mode='lines+markers'

    )

    data = [trace]

    layout = dict(title = title,

              xaxis = dict(title = xtitle), yaxis = dict(title = ytitle),

             )

    fig = dict(data=data, layout=layout)

    iplot(fig, filename='lines')
count_all = historical_trans_df.groupby('dayofweek')['purchase_amount'].agg(['sum'])

count_all.columns = ["Total"]

count_all = count_all.sort_index()

plot_scatter_data(count_all['Total'],'Day of week', 'Total','Total sum of purchase per day of week','green')
count_all = historical_trans_df.groupby('weekofyear')['purchase_amount'].agg(['sum'])

count_all.columns = ["Total"]

count_all = count_all.sort_index()

plot_scatter_data(count_all['Total'],'Week of year', 'Total','Total sum of purchase per Week of Year','red')
count_all = historical_trans_df.groupby('month')['purchase_amount'].agg(['sum'])

count_all.columns = ["Total"]

count_all = count_all.sort_index()

plot_scatter_data(count_all['Total'],'Month', 'Total','Total sum of purchase per month','blue')
plot_bar(get_categories(new_merchant_trans_df,'category_1'),

             'Category 1 distribution', 'Category 1', 'Number of records','gold')
plot_bar(get_categories(new_merchant_trans_df,'category_2'),

             'Category 2 distribution', 'Category 2', 'Number of records','tomato')
plot_bar(get_categories(new_merchant_trans_df,'category_3'),

             'Category 3 distribution', 'Category 3', 'Number of records','magenta')
plot_bar(get_categories(new_merchant_trans_df,'city_id'),

             'City ID distribution', 'City ID', 'Number of records','brown')
plot_bar(get_categories(new_merchant_trans_df,'merchant_category_id'),

             'Merchant category ID distribution', 'Merchant category ID', 'Number of records','green')
plot_bar(get_categories(new_merchant_trans_df,'state_id'),

             'State ID distribution', 'State ID', 'Number of records','darkblue')
plot_bar(get_categories(new_merchant_trans_df,'subsector_id'),

             'Subsector ID distribution', 'Subsector ID', 'Number of records','darkgreen')
new_merchant_trans_df['purchase_date'] = pd.to_datetime(new_merchant_trans_df['purchase_date'])

new_merchant_trans_df['month'] = new_merchant_trans_df['purchase_date'].dt.month

new_merchant_trans_df['dayofweek'] = new_merchant_trans_df['purchase_date'].dt.dayofweek

new_merchant_trans_df['weekofyear'] = new_merchant_trans_df['purchase_date'].dt.weekofyear
count_all = new_merchant_trans_df.groupby('month')['purchase_amount'].agg(['sum'])

count_all.columns = ["Total"]

count_all = count_all.sort_index()

plot_scatter_data(count_all['Total'],'Month', 'Total','Total sum of purchase per month','red')
count_all = new_merchant_trans_df.groupby('dayofweek')['purchase_amount'].agg(['sum'])

count_all.columns = ["Total"]

count_all = count_all.sort_index()

plot_scatter_data(count_all['Total'],'Day of week', 'Total','Total sum of purchase per day of week','magenta')
count_all = new_merchant_trans_df.groupby('weekofyear')['purchase_amount'].agg(['sum'])

count_all.columns = ["Total"]

count_all = count_all.sort_index()

plot_scatter_data(count_all['Total'],'Week of year', 'Total','Total sum of purchase per week of year','darkblue')
def plot_purchase_amount_distribution(data_df, var):

    hist_data = []

    varall = list(data_df.groupby([var])[var].nunique().index)

    for i, varcrt in enumerate(varall):

        classcrt = np.log(data_df[data_df[var] == varcrt]['purchase_amount'] + 1)

        hist_data.append(classcrt)

    fig = ff.create_distplot(hist_data, varall, show_hist=False, show_rug=False)

    fig['layout'].update(title='Purchase amount (log) variable density plot group by {}'.format(var), xaxis=dict(title='log(purchase_amount + 1)'))

    iplot(fig, filename='dist_only')
plot_purchase_amount_distribution(new_merchant_trans_df,'category_1')
plot_purchase_amount_distribution(new_merchant_trans_df,'category_2')
plot_purchase_amount_distribution(new_merchant_trans_df,'category_3')
plot_purchase_amount_distribution(new_merchant_trans_df,'state_id')
merchant_df.head(3)
plot_bar(get_categories(merchant_df,'merchant_category_id'),

             'Merchant category ID distribution', 'Merchant category ID', 'Number of records','darkblue')
plot_bar(get_categories(merchant_df,'subsector_id'),

             'Subsector ID distribution', 'Subsector ID', 'Number of records','blue')
plot_bar(get_categories(merchant_df,'category_1'),

             'Category 1 distribution', 'Category 1', 'Number of records','lightblue')
plot_bar(get_categories(merchant_df,'category_2'),

             'Category 2 distribution', 'Category 2', 'Number of records','lightgreen')
plot_bar(get_categories(merchant_df,'category_4'),

             'Category 4 distribution', 'Category 4', 'Number of records','tomato')
plot_bar(get_categories(merchant_df,'most_recent_sales_range'),

             'Most recent sales range distribution', 'Most recent sales range', 'Number of records','red')
plot_bar(get_categories(merchant_df,'most_recent_purchases_range'),

             'Most recent sales purchases distribution', 'Most recent purchases range', 'Number of records','magenta')
plot_bar(get_categories(merchant_df,'city_id'),

             'City ID distribution', 'City ID', 'Number of records','brown')
plot_bar(get_categories(merchant_df,'state_id'),

             'State ID distribution', 'State ID', 'Number of records','orange')
def plot_distribution(df,feature,color):

    fig, (ax1, ax2) = plt.subplots(ncols=2, figsize=(12,6))

    s = sns.boxplot(ax = ax1, data = df[feature].dropna(),color=color,showfliers=True)

    s.set_title("Distribution of %s (with outliers)" % feature)

    s = sns.boxplot(ax = ax2, data = df[feature].dropna(),color=color,showfliers=False)

    s.set_title("Distribution of %s (no outliers)" % feature)

    plt.show()
plot_distribution(merchant_df, "numerical_1", "blue")
plot_distribution(merchant_df, "numerical_2", "green")
plot_distribution(merchant_df, "avg_sales_lag3", "blue")
plot_distribution(merchant_df, "avg_sales_lag6", "green")
plot_distribution(merchant_df, "avg_sales_lag12", "green")
def get_logger():

    FORMAT = '[%(levelname)s]%(asctime)s:%(name)s:%(message)s'

    logging.basicConfig(format=FORMAT)

    logger = logging.getLogger('main')

    logger.setLevel(logging.DEBUG)

    return logger
# reduce memory

def reduce_mem_usage(df, verbose=True):

    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']

    start_mem = df.memory_usage().sum() / 1024**2

    for col in df.columns:

        col_type = df[col].dtypes

        if col_type in numerics:

            c_min = df[col].min()

            c_max = df[col].max()

            if str(col_type)[:3] == 'int':

                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:

                    df[col] = df[col].astype(np.int8)

                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:

                    df[col] = df[col].astype(np.int16)

                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:

                    df[col] = df[col].astype(np.int32)

                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:

                    df[col] = df[col].astype(np.int64)

            else:

                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:

                    df[col] = df[col].astype(np.float16)

                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:

                    df[col] = df[col].astype(np.float32)

                else:

                    df[col] = df[col].astype(np.float64)

    end_mem = df.memory_usage().sum() / 1024**2

    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))

    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))

    return df
logger = get_logger()

#process NAs

logger.info('Start processing NAs')

#process NA2 for transactions

for df in [historical_trans_df, new_merchant_trans_df]:

    df['category_2'].fillna(1.0,inplace=True)

    df['category_3'].fillna('A',inplace=True)

    df['merchant_id'].fillna('M_ID_00a6ca8a8a',inplace=True)

    df['installments'].replace(-1, np.nan,inplace=True)

    df['installments'].replace(999, np.nan,inplace=True)

#define function for aggregation

def create_new_columns(name,aggs):

    return [name + '_' + k + '_' + agg for k in aggs.keys() for agg in aggs[k]]
logger.info('process historical and new merchant datasets')

for df in [historical_trans_df, new_merchant_trans_df]:

    df['purchase_date'] = pd.to_datetime(df['purchase_date'])

    df['year'] = df['purchase_date'].dt.year

    df['weekofyear'] = df['purchase_date'].dt.weekofyear

    df['month'] = df['purchase_date'].dt.month

    df['dayofweek'] = df['purchase_date'].dt.dayofweek

    df['weekend'] = (df.purchase_date.dt.weekday >=5).astype(int)

    df['hour'] = df['purchase_date'].dt.hour

    df['authorized_flag'] = df['authorized_flag'].map({'Y':1, 'N':0})

    df['category_1'] = df['category_1'].map({'Y':1, 'N':0})

    df['category_3'] = df['category_3'].map({'A':0, 'B':1, 'C':2})

    df['month_diff'] = ((datetime.datetime.today() - df['purchase_date']).dt.days)//30

    df['month_diff'] += df['month_lag']

logger.info('new features historical and new merchant datasets')

for df in [historical_trans_df, new_merchant_trans_df]:

    df['price'] = df['purchase_amount'] / df['installments']

    df['Christmas_Day_2017']=(pd.to_datetime('2017-12-25')-df['purchase_date']).dt.days.apply(lambda x: x if x > 0 and x < 100 else 0)

    df['Children_day_2017']=(pd.to_datetime('2017-10-12')-df['purchase_date']).dt.days.apply(lambda x: x if x > 0 and x < 100 else 0)

    df['Black_Friday_2017']=(pd.to_datetime('2017-11-24') - df['purchase_date']).dt.days.apply(lambda x: x if x > 0 and x < 100 else 0)

    df['Mothers_Day_2018']=(pd.to_datetime('2018-05-13')-df['purchase_date']).dt.days.apply(lambda x: x if x > 0 and x < 100 else 0)

    df['month_diff'] = ((datetime.datetime.today() - df['purchase_date']).dt.days)//30

    df['month_diff'] += df['month_lag']

    df['duration'] = df['purchase_amount']*df['month_diff']

    df['amount_month_ratio'] = df['purchase_amount']/df['month_diff']

logger.info('reduce memory usage for historical trans')

historical_trans_df = reduce_mem_usage(historical_trans_df)

logger.info('reduce memory usage for new merchant trans')

new_merchant_trans_df = reduce_mem_usage(new_merchant_trans_df)
#define aggregations with historical_trans_df

logger.info('Aggregate historical trans')

aggs = {}

for col in ['subsector_id','merchant_id','merchant_category_id', 'state_id', 'city_id']:

    aggs[col] = ['nunique']

for col in ['month', 'hour', 'weekofyear', 'dayofweek']:

    aggs[col] = ['nunique', 'mean', 'min', 'max']

aggs['purchase_amount'] = ['sum','max','min','mean','var', 'std']

aggs['installments'] = ['sum','max','min','mean','var', 'std']

aggs['purchase_date'] = ['max','min', 'nunique']

aggs['month_lag'] = ['max','min','mean','var','nunique']

aggs['month_diff'] = ['mean', 'min', 'max', 'var','nunique']

aggs['authorized_flag'] = ['sum', 'mean', 'nunique']

aggs['weekend'] = ['sum', 'mean', 'nunique']

aggs['year'] = ['nunique', 'mean']

aggs['category_1'] = ['sum', 'mean', 'min', 'max', 'nunique', 'std']

aggs['category_2'] = ['sum', 'mean', 'min', 'nunique', 'std']

aggs['category_3'] = ['sum', 'mean', 'min', 'nunique', 'std']

aggs['card_id'] = ['size', 'count']

aggs['Christmas_Day_2017'] = ['mean']

aggs['Children_day_2017'] = ['mean']

aggs['Black_Friday_2017'] = ['mean']

aggs['Mothers_Day_2018'] = ['mean']

for col in ['category_2','category_3']:

    historical_trans_df[col+'_mean'] = historical_trans_df.groupby([col])['purchase_amount'].transform('mean')

    historical_trans_df[col+'_min'] = historical_trans_df.groupby([col])['purchase_amount'].transform('min')

    historical_trans_df[col+'_max'] = historical_trans_df.groupby([col])['purchase_amount'].transform('max')

    historical_trans_df[col+'_sum'] = historical_trans_df.groupby([col])['purchase_amount'].transform('sum')

    historical_trans_df[col+'_std'] = historical_trans_df.groupby([col])['purchase_amount'].transform('std')

    aggs[col+'_mean'] = ['mean']

new_columns = create_new_columns('hist',aggs)

historical_trans_group_df = historical_trans_df.groupby('card_id').agg(aggs)

historical_trans_group_df.columns = new_columns

historical_trans_group_df.reset_index(drop=False,inplace=True)

historical_trans_group_df['hist_purchase_date_diff'] = (historical_trans_group_df['hist_purchase_date_max'] - historical_trans_group_df['hist_purchase_date_min']).dt.days

historical_trans_group_df['hist_purchase_date_average'] = historical_trans_group_df['hist_purchase_date_diff']/historical_trans_group_df['hist_card_id_size']

historical_trans_group_df['hist_purchase_date_uptonow'] = (datetime.datetime.today() - historical_trans_group_df['hist_purchase_date_max']).dt.days

historical_trans_group_df['hist_purchase_date_uptomin'] = (datetime.datetime.today() - historical_trans_group_df['hist_purchase_date_min']).dt.days

logger.info('reduce memory usage for historical trans')

historical_trans_df = reduce_mem_usage(historical_trans_df)

logger.info('Completed aggregate historical trans')
#merge with train, test

train_df = train_df.merge(historical_trans_group_df,on='card_id',how='left')

test_df = test_df.merge(historical_trans_group_df,on='card_id',how='left')

#cleanup memory

del historical_trans_group_df; gc.collect()
#define aggregations with new_merchant_trans_df

logger.info('Aggregate new merchant trans')

aggs = {}

for col in ['subsector_id','merchant_id','merchant_category_id','state_id', 'city_id']:

    aggs[col] = ['nunique']

for col in ['month', 'hour', 'weekofyear', 'dayofweek']:

    aggs[col] = ['nunique', 'mean', 'min', 'max']

aggs['purchase_amount'] = ['sum','max','min','mean','var','std']

aggs['installments'] = ['sum','max','min','mean','var','std']

aggs['purchase_date'] = ['max','min', 'nunique']

aggs['month_lag'] = ['max','min','mean','var', 'nunique']

aggs['month_diff'] = ['mean', 'max', 'min', 'var','nunique']

aggs['weekend'] = ['sum', 'mean', 'nunique']

aggs['year'] = ['nunique', 'mean']

aggs['category_1'] = ['sum', 'mean', 'min', 'nunique']

aggs['category_2'] = ['sum', 'mean', 'min', 'nunique']

aggs['category_3'] = ['sum', 'mean', 'min', 'nunique']

aggs['card_id'] = ['size', 'count']

aggs['Christmas_Day_2017'] = ['mean']

aggs['Children_day_2017'] = ['mean']

aggs['Black_Friday_2017'] = ['mean']

aggs['Mothers_Day_2018'] = ['mean']

for col in ['category_2','category_3']:

    new_merchant_trans_df[col+'_mean'] = new_merchant_trans_df.groupby([col])['purchase_amount'].transform('mean')

    new_merchant_trans_df[col+'_min'] = new_merchant_trans_df.groupby([col])['purchase_amount'].transform('min')

    new_merchant_trans_df[col+'_max'] = new_merchant_trans_df.groupby([col])['purchase_amount'].transform('max')

    new_merchant_trans_df[col+'_sum'] = new_merchant_trans_df.groupby([col])['purchase_amount'].transform('sum')

    new_merchant_trans_df[col+'_std'] = new_merchant_trans_df.groupby([col])['purchase_amount'].transform('std')

    aggs[col+'_mean'] = ['mean']

new_columns = create_new_columns('new_hist',aggs)

new_merchant_trans_group_df = new_merchant_trans_df.groupby('card_id').agg(aggs)

new_merchant_trans_group_df.columns = new_columns

new_merchant_trans_group_df.reset_index(drop=False,inplace=True)

new_merchant_trans_group_df['new_hist_purchase_date_diff'] = (new_merchant_trans_group_df['new_hist_purchase_date_max'] - new_merchant_trans_group_df['new_hist_purchase_date_min']).dt.days

new_merchant_trans_group_df['new_hist_purchase_date_average'] = new_merchant_trans_group_df['new_hist_purchase_date_diff']/new_merchant_trans_group_df['new_hist_card_id_size']

new_merchant_trans_group_df['new_hist_purchase_date_uptonow'] = (datetime.datetime.today() - new_merchant_trans_group_df['new_hist_purchase_date_max']).dt.days

new_merchant_trans_group_df['new_hist_purchase_date_uptomin'] = (datetime.datetime.today() - new_merchant_trans_group_df['new_hist_purchase_date_min']).dt.days

logger.info('reduce memory usage for new merchant trans')

new_merchant_trans_df = reduce_mem_usage(new_merchant_trans_df)

logger.info('Completed aggregate new merchant trans')
#merge with train, test

train_df = train_df.merge(new_merchant_trans_group_df,on='card_id',how='left')

test_df = test_df.merge(new_merchant_trans_group_df,on='card_id',how='left')

#clean-up memory

del new_merchant_trans_group_df; gc.collect()

del historical_trans_df; gc.collect()

del new_merchant_trans_df; gc.collect()
#process train

logger.info('Process train - outliers')

train_df['outliers'] = 0

train_df.loc[train_df['target'] < -30, 'outliers'] = 1

outls = train_df['outliers'].value_counts()

print("Outliers: {}".format(outls))

logger.info('Process train and test')

## process both train and test

for df in [train_df, test_df]:

    df['first_active_month'] = pd.to_datetime(df['first_active_month'])

    df['dayofweek'] = df['first_active_month'].dt.dayofweek

    df['weekofyear'] = df['first_active_month'].dt.weekofyear

    df['dayofyear'] = df['first_active_month'].dt.dayofyear

    df['quarter'] = df['first_active_month'].dt.quarter

    df['is_month_start'] = df['first_active_month'].dt.is_month_start

    df['month'] = df['first_active_month'].dt.month

    df['elapsed_time'] = (datetime.datetime.today() - df['first_active_month']).dt.days

    df['hist_first_buy'] = (df['hist_purchase_date_min'] - df['first_active_month']).dt.days

    df['hist_last_buy'] = (df['hist_purchase_date_max'] - df['first_active_month']).dt.days

    df['new_hist_first_buy'] = (df['new_hist_purchase_date_min'] - df['first_active_month']).dt.days

    df['new_hist_last_buy'] = (df['new_hist_purchase_date_max'] - df['first_active_month']).dt.days

    for f in ['hist_purchase_date_max','hist_purchase_date_min','new_hist_purchase_date_max',\

                     'new_hist_purchase_date_min']:

        df[f] = df[f].astype(np.int64) * 1e-9

    df['card_id_total'] = df['new_hist_card_id_size']+df['hist_card_id_size']

    df['card_id_cnt_total'] = df['new_hist_card_id_count']+df['hist_card_id_count']

    df['purchase_amount_total'] = df['new_hist_purchase_amount_sum']+df['hist_purchase_amount_sum']

    df['purchase_amount_mean'] = df['new_hist_purchase_amount_mean']+df['hist_purchase_amount_mean']

    df['purchase_amount_max'] = df['new_hist_purchase_amount_max']+df['hist_purchase_amount_max']

    for f in ['feature_1','feature_2','feature_3']:

        order_label = train_df.groupby([f])['outliers'].mean()

        df[f] = df[f].map(order_label)

    df['feature_sum'] = df['feature_1'] + df['feature_2'] + df['feature_3']

    df['feature_mean'] = df['feature_sum']/3

    df['feature_max'] = df[['feature_1', 'feature_2', 'feature_3']].max(axis=1)

    df['feature_min'] = df[['feature_1', 'feature_2', 'feature_3']].min(axis=1)

    df['feature_var'] = df[['feature_1', 'feature_2', 'feature_3']].std(axis=1)

##

train_columns = [c for c in train_df.columns if c not in ['card_id', 'first_active_month','target','outliers']]

target = train_df['target']

del train_df['target']

logger.info('Completed process train')
#model

##model params

logger.info('Prepare model')

param = {'num_leaves': 51,

         'min_data_in_leaf': 35,

         'objective':'regression',

         'max_depth': -1,

         'learning_rate': 0.008,

         "boosting": "gbdt",

         "feature_fraction": 0.85,

         "bagging_freq": 1,

         "bagging_fraction": 0.82,

         "bagging_seed": 42,

         "metric": 'rmse',

         "lambda_l1": 0.11,

         "verbosity": -1,

         "nthread": 4,

         "random_state": 2019}

#prepare fit model with cross-validation

folds = StratifiedKFold(n_splits=9, shuffle=True, random_state=2019)

oof = np.zeros(len(train_df))

predictions = np.zeros(len(test_df))

feature_importance_df = pd.DataFrame()

#run model

logger.info('Start running model')

for fold_, (trn_idx, val_idx) in enumerate(folds.split(train_df,train_df['outliers'].values)):

    strLog = "Fold {}".format(fold_)

    print(strLog)

    trn_data = lgb.Dataset(train_df.iloc[trn_idx][train_columns], label=target.iloc[trn_idx])#, categorical_feature=categorical_feats)

    val_data = lgb.Dataset(train_df.iloc[val_idx][train_columns], label=target.iloc[val_idx])#, categorical_feature=categorical_feats)

    num_round = 10000

    clf = lgb.train(param, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=200, early_stopping_rounds = 150)

    oof[val_idx] = clf.predict(train_df.iloc[val_idx][train_columns], num_iteration=clf.best_iteration)

    #feature importance

    fold_importance_df = pd.DataFrame()

    fold_importance_df["Feature"] = train_columns

    fold_importance_df["importance"] = clf.feature_importance()

    fold_importance_df["fold"] = fold_ + 1

    feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)

    #predictions

    predictions += clf.predict(test_df[train_columns], num_iteration=clf.best_iteration) / folds.n_splits

    logger.info(strLog)

strRMSE = "".format(np.sqrt(mean_squared_error(oof, target)))

print(strRMSE)
##plot the feature importance

logger.info("Feature importance plot")

cols = (feature_importance_df[["Feature", "importance"]]

        .groupby("Feature")

        .mean()

        .sort_values(by="importance", ascending=False)[:1000].index)

best_features = feature_importance_df.loc[feature_importance_df.Feature.isin(cols)]

plt.figure(figsize=(14,26))

sns.barplot(x="importance", y="Feature", data=best_features.sort_values(by="importance",ascending=False))

plt.title('LightGBM Features (averaged over folds)')

plt.tight_layout()

plt.savefig('lgbm_importances.png')
##submission

logger.info("Prepare submission")

sub_df = pd.DataFrame({"card_id":test_df["card_id"].values})

sub_df["target"] = predictions

sub_df.to_csv("submission.csv", index=False)


from transformers import GPT2LMHeadModel, GPT2Tokenizer
from tqdm import tqdm
from sacrebleu.metrics import BLEU
import pandas as pd
import torch
import argparse

parser = argparse.ArgumentParser('Metrics')
parser.add_argument('-device', default='cuda:2', type=str, help='device_of_model_calculation')
parser.add_argument('-target', default='data/detox/test_ru_RU.0', type=str, help='target file path')
parser.add_argument('-predict', default='data/outputs/mbart_en_data_ru_RU.0', type=str, help='prediction file path')
parser.add_argument('-df_res', default='result/df_res.csv', type=str, help='path to df with results')
parser.add_argument('-ppl_model_name', default='sberbank-ai/rugpt2large', type=str, help='model name from ppl')
parser.add_argument('-model_name', default='', type=str, help='name of model to log it in df_res')
parser.add_argument('-exper_params', default='sberbank-ai/rugpt2large', type=str, help='experiment params to log it in df_res')
opt = parser.parse_args()

device = opt.device

# device = 'cuda:2' if torch.cuda.is_available() else 'cpu'
# pred_path = 'data/outputs/mbart_en_data_ru_RU.0'
# target_path = 'data/detox/test_ru_RU.0'
# df_res_path = 'result/df_res.csv'
# model_name_or_path = "sberbank-ai/rugpt2large"

def load_data(pred_path, target_path):
    targets = []
    with open(target_path, 'r') as f:
        for i, line in enumerate(f.readlines()):
            targets.append(line)

    preds = []
    with open(pred_path, 'r') as f:
        for i, line in enumerate(f.readlines()):
            preds.append(line)

    return targets, preds

def transform_text(text):
    text = text.strip().replace('"', '').replace("'", "")
    text = '\n' + text + '.\n'
    return text

def calculate_perplexity(preds, targets, model=None, tokenizer=None,
                         model_name_or_path="sberbank-ai/rugpt2large", weight=True, and_diff=False):
    if model is None:
        print('loading model')
        model = GPT2LMHeadModel.from_pretrained(model_name_or_path).to(device)
    if tokenizer is None:
        print('loading tokenizer')
        tokenizer = GPT2Tokenizer.from_pretrained(model_name_or_path)

    mean_ppl = 0
    len_tockens = 0
    if and_diff:
        mean_ppl_diff = 0

    if len(preds) != len(targets):
        raise ValueError("Number of predicted texts should be equal number of original texts")

    for i in tqdm(range(len(preds))):

        pred, target = transform_text(preds[i]), transform_text(targets[i])

        input_ids_pred = tokenizer.encode(pred, return_tensors="pt").to(device)
        out_pred = model(input_ids_pred.to(device), labels=input_ids_pred.to(device))
        loss_pred = out_pred.loss.cpu().detach().numpy().tolist()

        input_ids_target = tokenizer.encode(pred, return_tensors="pt").to(device)
        out_target = model(input_ids_target.to(device), labels=input_ids_target.to(device))
        loss_target = out_target.loss.cpu().detach().numpy().tolist()

        if weight:
            len_tok = len(input_ids_target.flatten()) - 1
            mean_ppl += (loss_pred) * len_tok
            if and_diff:
                mean_ppl_diff += (loss_target - loss_pred) * len_tok
            len_tockens += len_tok
        else:
            mean_ppl += loss_target - loss_pred
            if and_diff:
                mean_ppl_diff += (loss_target - loss_pred)
            len_tockens += 1

    mean_ppl /= len_tockens

    if and_diff:
        mean_ppl_diff /= len_tockens
        return mean_ppl, mean_ppl_diff
    else:
        return mean_ppl

def calculate_metrics(targets, preds, ppl_model_name="sberbank-ai/rugpt2large"):
    model_ppl = GPT2LMHeadModel.from_pretrained(ppl_model_name).to(device)
    tokenizer_ppl = GPT2Tokenizer.from_pretrained(ppl_model_name)

    sta = 0  # TODO
    bleu = BLEU(force=True).corpus_score(preds, targets).score
    ppl, ppl_diff = calculate_perplexity(preds, targets, model=model_ppl,
                                         tokenizer=tokenizer_ppl, weight=True, and_diff=True)

    return sta, bleu, ppl, ppl_diff

def main_metrics(pred_path, target_path, df_res_path, model_name='', exp_details='',
                 ppl_model_name="sberbank-ai/rugpt2large", verbose=True):
    df_res = pd.read_csv(df_res_path)
    targets, preds = load_data(pred_path, target_path)
    sta, bleu, ppl, ppl_diff = calculate_metrics(targets, preds, ppl_model_name)

    new_line = dict(zip(df_res.columns, [model_name, exp_details, sta, bleu, ppl, ppl_diff]))

    df_res.append(new_line, ignore_index=True)
    df_res.to_csv('result/df_res.csv', index=None)

    if verbose:
        print(new_line)

if __name__ == '__main__':
    # main_metrics(pred_path, target_path, df_res_path)
    main_metrics(opt.predict, opt.target, opt.df_res, ppl_model_name=opt.ppl_model_name,
                 model_name=opt.model_name, exp_details=opt.exper_params)


from typing import List

from accounts.schemas import UserData
from payments.models import Order, OrderProduct
from payments.schemas import OrderData
from products.models import Product

class PaymentRepository:
    def create_order(
        self,
        amount: int,
        data: OrderData,
        user_account: UserData,
    ) -> Order:
        order = Order.objects.create(
            user_id=user_account.id,
            amount=amount,
            without_calling=data.without_calling,
            customer_email=data.customer_email,
            receiver_name=data.receiver_name,
            receiver_phone_number=data.receiver_phone_number,
            delivery_address=data.delivery_address,
            delivery_date=data.delivery_date,
            delivery_time=data.delivery_time,
            note=data.note,
            cash=data.cash,
            is_paid=False,
            delivering=data.delivering,
        )
        return order

    def get_order_products_from_db(
        self,
        products_slug: List[str],
    ) -> List[Product]:
        order_products = Product.objects.only('title', 'slug', 'price').filter(
            slug__in=[*products_slug],
        )
        return order_products

    def create_order_products(
        self,
        order_id: int,
        order_products: List[Product],
        products_with_count: dict,
    ) -> None:
        products_to_bulk_create = [
            OrderProduct(
                order_id=order_id,
                product=product,
                count=int(products_with_count.get(product.slug)),
            )
            for product in order_products
        ]
        OrderProduct.objects.bulk_create(products_to_bulk_create)


from setuptools import setup, find_packages

requires = [
    'PyYAML==5.1'
]

setup(
    name='swagger_to_uml',
    version='0.1',
    description='swagger_to_uml',
    classifiers=[
        "Programming Language :: Python"
    ],
    author='Niels Lohmann',
    author_email='mail@nlohmann.me',
    license='MIT',
    url='http://nlohmann.me',
    keywords='swagger uml plantuml',
    packages=find_packages(),
    include_package_data=True,
    zip_safe=False,
    install_requires=requires,
    tests_require=requires,
    scripts=['bin/swagger_to_uml']
)


import json
from plotly.graph_objs import Scattergeo, Layout
from plotly import offline

# Read file with Earthquakes in the World: last 24 file form
filename = 'data/quake.json'
with open(filename) as f:
    data_quake = json.load(f)

# Show data in better form
# readfile = 'data/read_quake.json'
# with open(readfile, 'w') as f:
#     json.dump(data_quake, f, indent=5)

# Create list with features which contain information about quakes
all_quake_dict = data_quake['features']

# var for cordinates quakes
cordinates_x, cordinates_y = [], []

for quake_dict in all_quake_dict:
    cordinate_x = quake_dict['geometry']['coordinates'][0]  # long nesting - debug helps
    cordinate_y = quake_dict['geometry']['coordinates'][1]
    cordinates_x.append(cordinate_x)
    cordinates_y.append(cordinate_y)

print(cordinates_x)
print(cordinates_y)

data = [Scattergeo(lon=cordinates_x, lat=cordinates_y)]  # Obj Scattergeo - placing points on the map
layout = Layout(title='Earthquakes in the World')

fig = {'data': data, 'layout': layout}
offline.plot(fig, filename='earthquakes.html')


#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Tests for `AbstractRetrieval` module."""

from collections import namedtuple
from nose.tools import assert_equal, assert_true

import scopus

# Base information
ab1 = scopus.AbstractRetrieval("2-s2.0-84930616647", view="FULL", refresh=True)
# Conference proceeding and no references
ab2 = scopus.AbstractRetrieval("2-s2.0-0029486824", view="FULL", refresh=True)
# Issuetitle and no affiliation
ab3 = scopus.AbstractRetrieval("2-s2.0-0001270077", view="FULL", refresh=True)
# Author group broken and author keywords
ab4 = scopus.AbstractRetrieval("2-s2.0-0000016206", view="FULL", refresh=True)
# ISBN
ab5 = scopus.AbstractRetrieval("2-s2.0-84919546381", view="FULL", refresh=True)
# Funding, sequencebanks, chemicals
ab6 = scopus.AbstractRetrieval("2-s2.0-85053478849", view="FULL", refresh=True)
# Contributor group
ab7 = scopus.AbstractRetrieval("2-s2.0-85050253030", view="FULL", refresh=True)

def test_abstract():
    expected = 'In this paper we propose a Bayesian analysis of seasonal '\
        'unit roots in quarterly observed time series. Seasonal unit root '\
        'processes are useful to describe economic series with changing '\
        'seasonal fluctuations. A natural alternative model for similar '\
        'purposes contains deterministic seasonal mean shifts instead of '\
        'seasonal stochastic trends. This leads to analysing seasonal unit '\
        'roots in the presence of mean shifts using Bayesian techniques. '\
        'Our method is illustrated using several simulated and empirical data.'
    assert_equal(ab4.abstract, expected)

def test_affiliation():
    aff = namedtuple('Affiliation', 'id name city country')
    expected = [aff(id='60027950', name='Carnegie Mellon University',
                    city='Pittsburgh', country='United States')]
    assert_equal(ab1.affiliation, expected)
    assert_equal(ab3.affiliation, None)

def test_aggregationType():
    assert_equal(ab1.aggregationType, 'Journal')
    assert_equal(ab2.aggregationType, 'Conference Proceeding')

def test_authkeywords():
    assert_equal(ab1.authkeywords, None)
    expected = ['Bayesian analysis', 'Seasonality',
                'Structural breaks', 'Unit roots']
    assert_equal(ab4.authkeywords, expected)

def test_authors():
    fields = 'auid indexed_name surname given_name affiliation_id'
    auth = namedtuple('Author', fields)
    expected = [auth(auid='7004212771', indexed_name='Kitchin J.R.',
                surname='Kitchin', given_name='John R.',
                affiliation_id=['60027950'])]
    assert_equal(ab1.authors, expected)

def test_authorgroup():
    fields = 'affiliation_id organization city_group country '\
             'auid indexed_name surname given_name'
    auth = namedtuple('Author', fields)
    expected = [auth(
        affiliation_id='60027950', city_group=None, country='United States',
        organization='Department of Chemical Engineering, Carnegie Mellon University',
        auid='7004212771', indexed_name='Kitchin J.',
        surname='Kitchin', given_name='John R.')]
    assert_equal(ab1.authorgroup, expected)

def test_citedby_count():
    expected = 5
    assert_true(ab1.citedby_count >= expected)

def test_citedby_link():
    expected = 'https://www.scopus.com/inward/citedby.uri?partnerID=HzOxMe3b'\
        '&scp=84930616647&origin=inward'
    assert_equal(ab1.citedby_link, expected)

def test_chemials():
    received = ab6.chemicals
    assert_true(isinstance(received, list))
    assert_equal(len(received), 2)
    chemical = namedtuple('Chemical', 'source chemical_name cas_registry_number')
    expected = chemical(source='esbd', chemical_name='magnesium',
                        cas_registry_number='7439-95-4')
    assert_true(expected in received)
    assert_equal(ab3.chemicals, None)

def test_confcode():
    assert_equal(ab2.confcode, '44367')

def test_confdate():
    assert_equal(ab2.confdate, ((1995, 12, 13), (1995, 12, 15)))

def test_conflocation():
    assert_equal(ab2.conflocation, 'New Orleans, LA, USA')

def test_confname():
    expected2 = "Proceedings of the 1995 34th IEEE Conference on Decision "\
                "and Control. Part 1 (of 4)"
    assert_equal(ab2.confname, expected2)
    assert_equal(ab3.confname, None)
    expected7 = '20th Symposium on Design, Test, Integration and Packaging '\
                'of MEMS and MOEMS, DTIP 2018'
    assert_equal(ab7.confname, expected7)

def test_confsponsor():
    assert_equal(ab2.confsponsor, 'IEEE')
    assert_equal(ab3.confsponsor, None)
    expected7 = ['ARTOV.IMM.CNR.IT', 'CMP.IMAG.FR', 'CNRS.FR',
                 'EPS.IEEE.ORG', 'LIRMM.FR']
    assert_equal(ab7.confsponsor, expected7)

def test_contributor_group():
    fields = 'given_name initials surname indexed_name role'
    pers = namedtuple('Contributor', fields)
    expected = pers(given_name='Romolo', initials='R.', surname='Marcelli',
                    indexed_name='Marcelli R.', role='edit')
    received = ab7.contributor_group
    assert_equal(len(received), 7)
    assert_true(expected in received)
    assert_equal(ab3.contributor_group, None)

def test_correspondence():
    fields = 'surname initials organization country city_group'
    corr = namedtuple('Correspondence', fields)
    expected2 = corr(surname='Boukas', initials='E.K.',
        organization='Ecole Polytechnique de Montreal', country='Canada',
        city_group='Montreal')
    assert_equal(ab2.correspondence, expected2)
    assert_equal(ab3.correspondence, None)

def test_coverDate():
    assert_true(ab1.coverDate, '2015-06-05')

def test_description():
    expected = 'In this paper we propose a Bayesian analysis of seasonal '\
               'unit roots in quarterly observed time series. Seasonal unit '\
               'root processes are useful to describe economic series with '\
               'changing seasonal fluctuations. A natural alternative model '\
               'for similar purposes contains deterministic seasonal mean '\
               'shifts instead of seasonal stochastic trends. This leads to '\
               'analysing seasonal unit roots in the presence of mean '\
               'shifts using Bayesian techniques. Our method is illustrated '\
               'using several simulated and empirical data.'
    assert_equal(ab4.description, expected)

def test_doi():
    assert_equal(ab1.doi, '10.1021/acscatal.5b00538')
    assert_equal(ab4.doi, None)

def test_eid():
    assert_equal(ab1.eid, '2-s2.0-84930616647')

def test_endingPage():
    assert_equal(ab1.endingPage, '3899')

def test_funding():
    received = ab6.funding
    assert_true(isinstance(received, list))
    assert_equal(len(received), 1)
    fund = namedtuple('Funding', 'agency string id acronym country')
    expected6 = fund(agency=None, string='CNRT “Nickel et son Environnement',
        acronym=None, id=None, country=None)
    assert_true(expected6 in received)
    assert_equal(ab5.funding, None)

def test_funding_text():
    e = 'The authors gratefully acknowledge CNRT “Nickel et son '\
        'Environnement” for providing the financial support. The results '\
        'reported in this publication are gathered from the CNRT report '\
        '“Ecomine BioTop”. Appendix A'
    assert_equal(ab6.funding_text, e)

def test_get_html():
    e = '@article{Kaufmann1991FairnessPricing,\n  author = {Patrick J. '\
        'Kaufmann and Gwen Ortmeyer and N. Craig Smith},\n  title = {{'\
        'Fairness in consumer pricing}},\n  journal = {Journal of Consumer '\
        'Policy},\n  year = {1991},\n  volume = {14},\n  number = {2},\n  '\
        'pages = {117-140},\n  doi = {10.1007/BF00381915}}'
    assert_equal(ab3.get_bibtex(), e)

def test_get_latex():
    e = 'Philip Hans Franses, Henk Hoekh and Richard Paap, \\textit{'\
        'Bayesian analysis of seasonal unit roots and seasonal mean shifts},'\
        ' Journal of Econometrics, \\textbf{78(2)}, pp. 359-380 (1997).\\'\
        'href{https://www.scopus.com/inward/record.uri?partnerID=HzOxMe3b&'\
        'scp=0000016206&origin=inward}{scopus:2-s2.0-0000016206}.'
    assert_equal(ab4.get_latex(), e)

def test_get_ris():
    e = 'TY  - JOUR\nTI  - Examples of effective data sharing in scientific '\
        'publishing\nJO  - ACS Catalysis\nVL  - 5\nDA  - 2015-06-05\nPY  - '\
        '2015\nSP  - 3894-3899\nAU  - Kitchin J.R.\nDO  - 10.1021/'\
        'acscatal.5b00538\nUR  - https://doi.org/10.1021/acscatal.5b00538\n'\
        'IS  - 6\nER  - \n\n'
    assert_equal(ab1.get_ris(), e)

def test_get_html():
    e = 'Patrick J. Kaufmann, Gwen Ortmeyer and N. Craig Smith, Fairness in consumer pricing, Journal of Consumer '\
        'Policy, 14(2), pp. 117-140, (1991). doi:10.1007/BF00381915.'
    assert_equal(ab3.get_html(), e)

def test_isbn():
    assert_equal(ab3.isbn, None)
    assert_equal(ab5.isbn, ('0203881486', '9780203881484'))

def test_issn():
    assert_equal(ab1.issn, '21555435')
    assert_equal(ab5.issn, None)

def test_identifier():
    assert_equal(ab1.identifier, '84930616647')

def test_idxterms():
    expected = ['Control variables', 'Cost function',
                'Hamilton-Jacobi-Isaacs equation', 'Machine capacity',
                'Stochastic manufacturing systems', 'Value function']
    assert_equal(ab2.idxterms, expected)
    assert_equal(ab4.idxterms, None)

def test_issueIdentifier():
    assert_equal(ab1.issueIdentifier, '6')

def test_issuetitle():
    assert_equal(ab3.issuetitle, 'Law, Economics and Behavioural Sciences')

def test_language():
    assert_equal(ab1.language, 'eng')

def test_pageRange():
    assert_equal(ab1.pageRange, '3894-3899')

def test_publicationName():
    assert_equal(ab1.publicationName, 'ACS Catalysis')

def test_publisher():
    assert_equal(ab1.publisher, 'American Chemical Society')

def test_publisheraddress():
    assert_equal(ab2.publisheraddress, 'Piscataway, NJ, United States')

def test_refcount():
    assert_equal(ab4.refcount, '18')

def test_references():
    fields = 'position id doi title authors sourcetitle publicationyear '\
             'volume issue first last text fulltext'
    ref = namedtuple('Reference', fields)
    fulltext1 = 'Implementing Reproducible Research; Stodden, V.; Leisch, '\
                'F.; Peng, R. D., Eds., Chapman and Hall/CRC: London, 2014.'
    expected1 = ref(position='22', id='85055586929', doi=None, title=None,
        authors=['Stodden, V.', 'Leisch, F.', 'Peng, R.D.'], fulltext=fulltext1,
        sourcetitle='Implementing Reproducible Research',
        publicationyear='2014', volume=None, issue=None, first=None,
        last=None, text='Eds. Chapman and Hall/CRC: London.',)
    assert_equal(ab1.references[-1], expected1)
    assert_equal(ab2.references, None)
    fulltext4 = 'Chib, S., 1995, Marginal likelihood from the Gibbs output, '\
                'Journal of the American Statistical Association 90, 1313-1321.'
    expected4 = ref(position='1', id='0041974049', doi=None,
        title='Marginal likelihood from the Gibbs output', authors=['Chib, S.'],
        sourcetitle='Journal of the American Statistical Association',
        publicationyear='1995', volume='90', issue=None, first='1313',
        last='1321', text=None, fulltext=fulltext4)
    assert_equal(ab4.references[0], expected4)

def test_scopus_link():
    expected = 'https://www.scopus.com/inward/record.uri?partnerID=HzOxMe3b&'\
        'scp=84930616647&origin=inward'
    assert_equal(ab1.scopus_link, expected)

def test_self_link():
    expected = 'https://api.elsevier.com/content/abstract/scopus_id/84930616647'
    assert_equal(ab1.self_link, expected)

def test_sequencebank():
    received = ab6.sequencebank
    assert_true(isinstance(received, list))
    bank = namedtuple('Chemical', 'name sequence_number type')
    expected = bank(name='GENBANK', type='submitted',
                    sequence_number='MH150839:MH150870')
    assert_true(expected in received)
    assert_equal(ab3.sequencebank, None)

def test_source_id():
    assert_equal(ab1.source_id, '19700188320')

def test_sourcetitle_abbreviation():
    assert_equal(ab1.sourcetitle_abbreviation, 'ACS Catal.')

def test_srctype():
    assert_equal(ab1.srctype, 'j')
    assert_equal(ab2.srctype, 'p')

def test_startingPage():
    assert_equal(ab1.startingPage, '3894')

def test_subject_areas():
    area = namedtuple('Area', 'area abbreviation code')
    expected = [area(area='Catalysis', abbreviation='CENG', code='1503'),
                area(area='Chemistry (all)', abbreviation='CHEM', code='1600')]
    assert_equal(ab1.subject_areas, expected)

def test_title():
    expected = 'Examples of effective data sharing in scientific publishing'
    assert_equal(ab1.title, expected)

def test_url():
    expected = 'https://api.elsevier.com/content/abstract/scopus_id/84930616647'
    assert_equal(ab1.url, expected)

def test_volume():
    assert_equal(ab1.volume, '5')

def test_website():
    assert_equal(ab1.website, 'http://pubs.acs.org/page/accacs/about.html')
    assert_equal(ab2.website, None)


from fastapi import APIRouter,Depends
from sqlalchemy.orm import Session

from core.config import get_db

router=APIRouter(
    prefix='/items',
    tags=['Items']
)

@router.get('/')
def get_user(db:Session =Depends(get_db)):
    return 'Items'


# -------------------------------------------------------------------------- #
# IMPORTAÇÕES

from tkinter import Tk, Frame, Label, Button, Entry
from tkinter.ttk import Style
from PIL import Image, ImageTk

# -------------------------------------------------------------------------- #
# CONSTANTES

COLOR0 = '#2e2d2b'  # Preto
COLOR1 = '#feffff'  # Branco
COLOR2 = '#4fa882'  # Verde
COLOR3 = '#28576b'  # Valor
COLOR4 = '#403d3d'  # Letra
COLOR5 = '#e06636'  # Profit
COLOR6 = '#038cfc'  # Azul
COLOR7 = '#3fbfb9'  # Verde
COLOR8 = '#263238'  # + Verde
COLOR9 = '#e9edf5'  # Branco

# -------------------------------------------------------------------------- #
# FUNÇÕES

def output_calculations(hours, days, weeks, months, years):
    """Função para mostrar resultados dos cálculos."""
    hour_output_label['text'] = f' Salário por hora: R$ {hours:,.2f}'
    day_output_label['text'] = f' Salário diário: R$ {days:,.2f}'
    week_output_label['text'] = f' Salário semanal: R$ {weeks:,.2f}'
    month_output_label['text'] = f' Salário mensal: R$ {months:,.2f}'
    annual_output_label['text'] = f' Salário anual: R$ {years:,.2f}'

def output_entry_info(name, total_hours_day, work_days, annual_salary):
    """Função para mostrar os dados digitados pelo usuário."""
    output_name_label['text'] = name
    output_hours_label['text'] = f'Horas de trabalho ' \
        f'por dia: {total_hours_day}'
    output_days_label['text'] = f'Dias de trabalho por semana: {work_days}'
    output_annual_salary_label['text'] = f'R$ {annual_salary:,.2f}'

def calculate_salary():
    """Função para calcular o salario."""
    # Pega o input do usuário
    name = name_entry.get().title()
    total_hours_day = int(hours_entry.get())
    work_days = int(days_entry.get())
    annual_salary = float(annual_salary_entry.get())

    # Calculando total horas de trabalho por semana
    total_hours_work_week = total_hours_day * work_days

    # Calculando total horas de trabalho por ano
    # 52 - as semanas no ano
    total_hours_work_year = 52 * total_hours_work_week

    # Calculando o salário por horas
    hourly_pay = annual_salary / total_hours_work_year

    # Calculando salário diário
    daily_pay = hourly_pay * total_hours_day

    # Calculando salário semanal
    weekly_pay = annual_salary / 52

    # Calculando salário mensal
    monthly_pay = annual_salary / 12

    # Calculando salário bruto anual
    annual_pay = monthly_pay * 12

    output_entry_info(name, total_hours_day, work_days, annual_salary)
    output_calculations(
        hourly_pay, daily_pay,
        weekly_pay, monthly_pay, annual_pay
    )

# -------------------------------------------------------------------------- #
# JANELA

window = Tk()
window.title('')
window.geometry('580x350')
window.resizable(0, 0)
window.configure(bg=COLOR9)

style = Style(window)
style.theme_use('clam')

# -------------------------------------------------------------------------- #
# FRAMES

upper_frame = Frame(window, width=580, height=60, pady=5, bg=COLOR1)
upper_frame.grid(row=0, column=0)

lower_frame = Frame(window, width=580, height=290, pady=0, bg=COLOR1)
lower_frame.grid(row=1, column=0)

# -------------------------------------------------------------------------- #
# LABELS

# Logo e título do app.
app_image_logo = Image.open('imagens/salary.png')
app_image_logo = app_image_logo.resize((25, 25))
app_image_logo = ImageTk.PhotoImage(app_image_logo)
app_label = Label(
    upper_frame, text='Calculadora de Salário', image=app_image_logo,
    width=600, compound='left', relief='raised', anchor='nw',
    font=('Verdana 15'), bg=COLOR1, fg=COLOR4
)
app_label.place(x=0, y=0)

# Labels para pedir input do usuario.
# Nome, horas de trabalho, dias de trabalho e
# salário anual.
name_label = Label(
    lower_frame, text='Escreva o seu nome completo',
    anchor='nw', font=('Roboto 10'),
    bg=COLOR1, fg=COLOR4
)
name_label.place(x=10, y=10)

hours_label = Label(
    lower_frame, text='Quantas horas você trabalha por dia?',
    anchor='nw', font=('Roboto 10'),
    bg=COLOR1, fg=COLOR4
)
hours_label.place(x=10, y=40)

days_label = Label(
    lower_frame, text='Quantos dias você trabalha por semana?',
    anchor='nw', font=('Roboto 10'),
    bg=COLOR1, fg=COLOR4
)
days_label.place(x=10, y=70)

annual_salary_label = Label(
    lower_frame, text='Qual é o seu salário anual?',
    anchor='nw', font=('Roboto 10'),
    bg=COLOR1, fg=COLOR4
)
annual_salary_label.place(x=10, y=100)

# Label e imagem que fica por cima do botão 'CALCULAR'
calculate_image_logo = Image.open('imagens/salary.png')
calculate_image_logo = calculate_image_logo.resize((90, 90))
calculate_image_logo = ImageTk.PhotoImage(calculate_image_logo)
calculate_label = Label(
    lower_frame, image=calculate_image_logo,
    bg=COLOR1, fg=COLOR4
)
calculate_label.place(x=470, y=0)

# Labels que ficam na parte inferior esquerda do app.
# Espelham o input do usuário, nome, horas, dias e salário
output_name_label = Label(
    lower_frame, width=30, pady=5, padx=5,
    relief='raised', anchor='nw',
    font=('Roboto 10'), bg=COLOR9, fg=COLOR0
)
output_name_label.place(x=10, y=150)

output_hours_label = Label(
    lower_frame, width=30, pady=5, padx=5,
    relief='raised', anchor='nw',
    font=('Roboto 10'), bg=COLOR9, fg=COLOR0
)
output_hours_label.place(x=10, y=179)

output_days_label = Label(
    lower_frame, width=30, pady=5, padx=5,
    relief='raised', anchor='nw',
    font=('Roboto 10'), bg=COLOR9, fg=COLOR0
)
output_days_label.place(x=10, y=208)

output_annual_salary_label = Label(
    lower_frame, width=30, pady=5, padx=5,
    relief='raised', anchor='nw',
    font=('Roboto 10 bold'), bg=COLOR9, fg=COLOR0
)
output_annual_salary_label.place(x=10, y=237)

# Labels que ficam na parte inferior direita do app.
# Mostram o resultado dos cálculos, salário por hora,
# por dia, por semana, mês e por ano.
hour_output_logo = Image.open('imagens/yellow-circle.png')
hour_output_logo = hour_output_logo.resize((20, 20))
hour_output_logo = ImageTk.PhotoImage(hour_output_logo)
hour_output_label = Label(
    lower_frame, image=hour_output_logo,
    width=300, compound='left', anchor='nw',
    font=('Roboto 10'), bg=COLOR1, fg=COLOR0
)
hour_output_label.place(x=300, y=145)

day_output_logo = Image.open('imagens/blue-circle.png')
day_output_logo = day_output_logo.resize((20, 20))
day_output_logo = ImageTk.PhotoImage(day_output_logo)
day_output_label = Label(
    lower_frame, image=day_output_logo,
    width=300, compound='left', anchor='nw',
    font=('Roboto 10'), bg=COLOR1, fg=COLOR0
)
day_output_label.place(x=300, y=170)

week_output_logo = Image.open('imagens/gray-circle.png')
week_output_logo = week_output_logo.resize((20, 20))
week_output_logo = ImageTk.PhotoImage(week_output_logo)
week_output_label = Label(
    lower_frame, image=week_output_logo,
    width=300, compound='left', anchor='nw',
    font=('Roboto 10'), bg=COLOR1, fg=COLOR0
)
week_output_label.place(x=300, y=195)

month_output_logo = Image.open('imagens/red-circle.png')
month_output_logo = month_output_logo.resize((20, 20))
month_output_logo = ImageTk.PhotoImage(month_output_logo)
month_output_label = Label(
    lower_frame, image=month_output_logo,
    width=300, compound='left', anchor='nw',
    font=('Roboto 10'), bg=COLOR1, fg=COLOR0
)
month_output_label.place(x=300, y=220)

annual_output_logo = Image.open('imagens/green-circle.png')
annual_output_logo = annual_output_logo.resize((20, 20))
annual_output_logo = ImageTk.PhotoImage(annual_output_logo)
annual_output_label = Label(
    lower_frame, image=annual_output_logo,
    width=300, compound='left', anchor='nw',
    font=('Roboto 10'), bg=COLOR1, fg=COLOR0
)
annual_output_label.place(x=300, y=245)

# -------------------------------------------------------------------------- #
# ENTRIES

name_entry = Entry(lower_frame, width=25, justify='center', relief='solid')
name_entry.place(x=200, y=11)

hours_entry = Entry(lower_frame, width=15, justify='center', relief='solid')
hours_entry.place(x=290, y=41)

days_entry = Entry(lower_frame, width=15, justify='center', relief='solid')
days_entry.place(x=290, y=71)

annual_salary_entry = Entry(
    lower_frame, width=15, justify='center', relief='solid'
)
annual_salary_entry.place(x=290, y=101)

# -------------------------------------------------------------------------- #
# BOTÕES

button_image_logo = Image.open('imagens/salary.png')
button_image_logo = button_image_logo.resize((20, 20))
button_image_logo = ImageTk.PhotoImage(button_image_logo)
calculate_button = Button(
    lower_frame, text='CALCULAR', image=button_image_logo,
    compound='left', relief='raised', overrelief='ridge',
    anchor='ne', font=('Roboto 8 bold'), bg=COLOR1, fg=COLOR4,
    command=calculate_salary
)
calculate_button.place(x=450, y=92)

# -------------------------------------------------------------------------- #
# LOOP

window.mainloop()


'''
Leetcode 24 submission
Your runtime beats 90.27 % of python3 submissions.
'''
from collections import deque as _deque

class ListNode:
    def __init__(self, x):
        self.x = x
        self.next = None

def solve(head):
    # simple return cases
    if head and head.next:
        deque = _deque()
        deque.append(head)
        deque.append(head.next)
        head = head.next
    else:
        return head

    while len(deque) >= 2:
        # swap the latest pair's pointer
        deque[-2].next, deque[-1].next = deque[-1].next, deque[-2]

        # swap them in the deque so they make sequential sense
        deque[-1], deque[-2] = deque[-2], deque[-1]

        # update the rightmost of the previous pair as it's neighbor changed
        if len(deque) > 2:
            deque[-3].next = deque[-2]

        # the leftmost element is now totally complete, so we pop it
        deque.popleft()

        # enqueue iff another non-null pair to swap is possible
        if deque[-1].next and deque[-1].next.next:
            deque.append(deque[-1].next)
            deque.append(deque[-1].next)
        else:
            break

    return head

def execute_tests():
    simple = ListNode(1)
    simple.next = ListNode(2)
    simple.next.next = ListNode(3)
    simple.next.next.next = ListNode(4)

    ans = "2143"

    for test_case in [(simple, ans)]:
        test(*test_case)

def test(head, expected):
    answer = solve(head)
    print(as_string((answer)))
    assert as_string(answer) == expected

def as_string(head):
    s = ''
    while head:
        s += str(head.x)
        head = head.next
    return s

def main():
    return execute_tests()

if __name__ == '__main__':
    main()


#!/usr/bin/env python
# -*- coding: utf-8 -*-

# @author: x.huang
# @date:09/04/18
import logging
import time

import tornado
from tornado import httputil
from tornado import ioloop, gen
from tornado import options
from tornado import web
from tornado.options import define, options, parse_command_line

UPLOAD_DIR = 'public'
define("port", default=8181, help="Run server on a specific port", type=int)

def save_file(file: httputil.HTTPFile, file_path):
    with open(file_path, 'wb+') as f:
        f.write(file.body)

class UploadHandler(web.RequestHandler):
    @gen.coroutine
    def post(self, *args, **kwargs):
        file = self.request.files.get('file')[0]  # type: httputil.HTTPFile
        now = time.time()
        file_path = "%s/%s_%s" % (UPLOAD_DIR, file.filename, now)
        save_file(file, file_path)
        self.write("%s upload success." % file_path)
        self.finish()

class HelloHandler(web.RequestHandler):
    @gen.coroutine
    def get(self):
        self.write({
            'msg': 'success'

        })
        self.finish()

class Application(web.Application):

    def __init__(self):
        handler_list = [
            (r'/upload', UploadHandler),
            (r'/hello', HelloHandler),
        ]
        settings = dict(
            debug=True,

        )
        web.Application.__init__(self, handler_list, **settings)

def start_server():
    options.logging = 'info'
    logging.basicConfig(level=logging.INFO)
    app = Application()
    parse_command_line()
    app.listen(options.port)

    loop = ioloop.IOLoop().current()
    try:
        logging.info('Tornado version {}'.format(tornado.version))
        logging.info('Tornado Server Start, listen on {}'.format(options.port))
        logging.info('Quite the Server with CONTROL-C.')
        loop.start()
    except KeyboardInterrupt:
        loop.close()

if __name__ == '__main__':
    start_server()


from MainWindow import MainWindow
from PyQt5.QtWidgets import QApplication

import os

folder_path = 'Output'
if not os.path.exists(folder_path):
    os.makedirs(folder_path)

app = QApplication([])

win = MainWindow()
win.show()

app.exec()


# # 1
# 9/10/2021 - 13/10/2021
# SPACE INVADER.SV
import pygame
from pygame import mixer
import math
import random

# Intialize the pygame
pygame.init()

# create the screen
screen = pygame.display.set_mode((600, 700))

# Background
moonImage = pygame.image.load("game_assets/game_images/moon.png")
moonX = 450
moonY = 0
moonY_change = 0.1

def moon(x, y):
    screen.blit(moonImage, (x, y))

meo2 = pygame.image.load('game_assets/game_images/white dot 2.jpg')
meo1 = pygame.image.load('game_assets/game_images/white dot.jpg')
meoType = [meo1, meo2]
meoImage = random.choice(meoType)
meoX = 0
meoY = 0
meoY_change = 2

def fall_meos(x, y):
    meoImage = random.choice(meoType)
    screen.blit(meoImage, (x + 190, y + 29))
    screen.blit(meoImage, (x + 553, y - 498))
    meoImage = random.choice(meoType)
    screen.blit(meoImage, (x + 220, y - 633))
    screen.blit(meoImage, (x + 497, y - 43))
    meoImage = random.choice(meoType)
    screen.blit(meoImage, (x + 178, y + 275))
    screen.blit(meoImage, (x + 544, y - 799))
    screen.blit(meoImage, (x + 219, y + 327))
    screen.blit(meoImage, (x + 358, y + 744))
    meoImage = random.choice(meoType)
    screen.blit(meoImage, (x + 435, y + 45))
    screen.blit(meoImage, (x + 580, y - 260))
    screen.blit(meoImage, (x + 90, y + 529))
    screen.blit(meoImage, (x + 53, y - 398))
    meoImage = random.choice(meoType)
    screen.blit(meoImage, (x + 20, y - 133))
    screen.blit(meoImage, (x + 497, y - 43))
    screen.blit(meoImage, (x + 78, y + 475))
    meoImage = random.choice(meoType)
    screen.blit(meoImage, (x + 544, y - 99))
    screen.blit(meoImage, (x + 219, y + 227))
    screen.blit(meoImage, (x + 358, y + 744))
    screen.blit(meoImage, (x + 35, y + 5))
    screen.blit(meoImage, (x + 580, y - 260))
    meoImage = random.choice(meoType)
    screen.blit(meoImage, (x + 190, y + 29))
    screen.blit(meoImage, (x + 53, y - 598))
    screen.blit(meoImage, (x + 220, y - 333))
    screen.blit(meoImage, (x + 97, y - 243))
    meoImage = random.choice(meoType)
    screen.blit(meoImage, (x + 178, y + 575))
    screen.blit(meoImage, (x + 544, y - 399))
    screen.blit(meoImage, (x + 19, y + 727))
    meoImage = random.choice(meoType)
    screen.blit(meoImage, (x + 358, y + 44))
    screen.blit(meoImage, (x + 35, y + 25))
    screen.blit(meoImage, (x + 580, y - 60))

# Sound
mixer.music.load("game_assets/game_sounds/deadite-ash-vs-evil-dead-song--8688.mp3")
# mixer.music.load("demon-slayer-8687.mp3")
mixer.music.play(-1)

pygame.display.set_caption("SPACE INVADERS.SV")
icon = pygame.image.load('game_assets/game_images/spaceship (1).png')
pygame.display.set_icon(icon)

# shipImage = pygame.image.load('game_assets/game_images/spaceship (1).png')
# shipImage = pygame.image.load('game_assets/game_images/spaceship (3).png')
shipImage = pygame.image.load('game_assets/game_images/spaceship (4).png')
shipX = 262
shipY = 635
shipX_change = 0

# Enemy
ufoType = []
ufoX = []
ufoY = []
ufoX_change = []
ufoY_change = []
num_of_ufos = 5

for i in range(num_of_ufos):
    # ufoImage = pygame.image.load('ufo.png')
    # ufo2Image = pygame.image.load('ufo 2.png')
    # ufosImage = [ufoImage, ufo2Image]
    ufoType.append(pygame.image.load('game_assets/game_images/ufo.png'))
    # ufoType.append('ufoType')
    ufoX.append(random.randint(0, 535))
    ufoY.append(random.randint(0, 10))
    ufoX_change.append(0.3)
    ufoY_change.append(0.3)

# Bullet

# Ready - You can't see the bullet on the screen
# Fire - The bullet is currently moving

'''bulletImg = pygame.image.load('game_assets/game_images/bullet.png')
bulletX = 0
bulletY = 480
bulletX_change = 0
bulletY_change = 10
bullet_state = "ready"'''
bulletImage2 = pygame.image.load('game_assets/game_images/laser1.png')
bulletImage1 = pygame.image.load('game_assets/game_images/laser.png')
bulletType = [bulletImage1, bulletImage2]
bulletImage = random.choice(bulletType)
bulletX = 0
bulletY = 635
bulletY_change = 2
bullet_state = "ready"

# Score

score_value = 0
font = pygame.font.Font('freesansbold.ttf', 32)

textX = 10
testY = 10

# Game Over
over_font = pygame.font.Font('freesansbold.ttf', 30)

def show_score(x, y):
    score = font.render("Score : " + str(score_value), True, (255, 255, 255))
    screen.blit(score, (x, y))

def game_over_text():
    over_text = over_font.render("GAME OVER", True, (255, 255, 255))
    screen.blit(over_text, (200, 250))

def ship(x, y):
    screen.blit(shipImage, (x, y))

def ufo(x, y, i):
    screen.blit(ufoType[i], (x, y))

def fire_bullet(x, y):
    global bullet_state
    bullet_state = "fire"
    bulletImage = random.choice(bulletType)
    screen.blit(bulletImage, (x - 2, y))
    screen.blit(bulletImage, (x + 34, y))

def isCollision(ufoX, ufoY, bulletX, bulletY):
    distance = math.sqrt(math.pow(ufoX - bulletX, 2) + (math.pow(ufoY - bulletY, 2)))
    if distance < 50:
        return True
    else:
        return False

# Game Loop
running = True
while running:

    # RGB = Red, Green, Blue
    screen.fill((0, 0, 0))
    # Background Image
    moon(moonX, moonY)
    moonY += moonY_change
    if moonY > 900:
        moonY = -100

    fall_meos(meoX, meoY)
    meoY += meoY_change
    if meoY > 800:
        meoY = 0
        meoImage = random.choice(meoType)
    # screen.blit(background, (0, 0))

    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            running = False

        # if keystroke is pressed check whether its right or left
        if event.type == pygame.KEYDOWN:
            if event.key == pygame.K_LEFT:
                shipX_change = -0.7
            if event.key == pygame.K_RIGHT:
                shipX_change = 0.7
            if event.key == pygame.K_SPACE:
                if bullet_state == "ready":
                    bulletSound = mixer.Sound("game_assets/game_sounds/3_12.wav")
                    bulletSound.play()
                    # Get the current x cordinate of the spaceship
                    bulletX = shipX
                    fire_bullet(bulletX, bulletY)

        if event.type == pygame.KEYUP:
            if event.key == pygame.K_LEFT or event.key == pygame.K_RIGHT:
                shipX_change = 0

    # 5 = 5 + -0.1 -> 5 = 5 - 0.1
    # 5 = 5 + 0.1

    shipX += shipX_change
    if shipX <= 0:
        shipX = 0
    elif shipX >= 536:
        shipX = 536

    # Enemy Movement
    for i in range(num_of_ufos):

        # Game Over
        if ufoY[i] > 600:
            for j in range(num_of_ufos):
                ufoY[j] = 610
            game_over_text()

            break

        ufoX[i] += ufoX_change[i]
        ufoY[i] += ufoY_change[i]

        if ufoX[i] <= 0:
            ufoX_change[i] = 0.15
        elif ufoX[i] >= 536:
            ufoX_change[i] = -0.15

        # Collision
        collision = isCollision(ufoX[i], ufoY[i], bulletX, bulletY)
        if collision:
            explosionSound = mixer.Sound("game_assets/game_sounds/QuantumTorp.mp3")
            explosionSound.play()
            bulletY = 635
            bullet_state = "ready"
            score_value += 1
            ufoX[i] = random.randint(0, 535)
            ufoY[i] = random.randint(-20, 10)

        ufo(ufoX[i], ufoY[i], i)

    # Bullet Movement
    if bulletY <= 0:
        bulletY = 635
        bullet_state = "ready"

    if bullet_state == "fire":
        fire_bullet(bulletX, bulletY)
        bulletY -= bulletY_change

    ship(shipX, shipY)
    show_score(textX, testY)
    pygame.display.update()


from functools import wraps

import redis
from django.apps import apps
from django.conf import settings
from django.http import HttpRequest, HttpResponse
from django.shortcuts import render
from django.utils import timezone
from django.views.decorators.http import require_safe

from config.celery import app as celery_app

CELERY_WORKERS: list = []
CELERY_WORKERS_LAST_REFRESH = timezone.now().timestamp()
CELERY_WORKERS_REFRESH_PERIOD = 900  # Once every 15 minutes refresh list of workers

def inspect_celery():
    """Return the Celery inspect object."""
    return celery_app.control.inspect(CELERY_WORKERS)

def check_app_status(func):
    """Check if Redis is up; if it is, check if Celery workers are up."""

    @wraps(func)
    def wrapped(*args, **kwargs):
        r = redis.Redis.from_url(settings.CELERY_BROKER_URL)
        try:
            r.ping()
        except redis.ConnectionError:
            return render(
                args[0],  # Response object
                "app_status.html",
                {"msg": "Redis message broker is not running..."},
            )

        global CELERY_WORKERS
        global CELERY_WORKERS_LAST_REFRESH

        # If we are due to refresh - wipe the list of workers to force full refresh
        if timezone.now().timestamp() - CELERY_WORKERS_LAST_REFRESH > CELERY_WORKERS_REFRESH_PERIOD:
            CELERY_WORKERS = []
            CELERY_WORKERS_LAST_REFRESH = timezone.now().timestamp()

        i = inspect_celery()
        # This is much faster if CELERY_WORKERS is set to current set of worker names
        ping_replies = i.ping()
        if (
            not ping_replies
            or len(ping_replies) < len(CELERY_WORKERS)
            or not all([r["ok"] == "pong" for r in ping_replies.values()])
        ):
            # Reset worker list - maybe they were redeployed?
            CELERY_WORKERS = []
            CELERY_WORKERS_LAST_REFRESH = timezone.now().timestamp()
            return render(
                args[0],  # Response object
                "app_status.html",
                {"msg": "Celery workers are not running..."},
            )

        # Update list of current workers
        CELERY_WORKERS = list(ping_replies.keys())
        return func(*args, **kwargs)

    return wrapped

@require_safe
def data_list(request: HttpRequest) -> HttpResponse:
    """Renders a count of all cached resources in the DB."""
    models = sorted(
        apps.get_app_config("core").get_models(),
        key=lambda model: model._meta.verbose_name,  # type: ignore[arg-type,return-value]
    )
    return render(
        request,
        "data.html",
        {
            "counts": [
                (model._meta.verbose_name_plural, model.objects.db_manager("read_only").count())
                for model in models
            ],
            "nbar": "data",  # Navbar identifier
        },
    )

@check_app_status
@require_safe
def tasks_list(request):
    inspect = inspect_celery()
    tasks = []
    for task in inspect.registered("__doc__")[CELERY_WORKERS[0]]:
        task_name, _, desc = task.partition(" ")
        desc = (
            desc.replace("[__doc__=", "").rstrip("]").replace("\n", "").strip() if desc else "N/A"
        )
        tasks.append({"name": task_name, "description": desc})

    return render(
        request,
        "task_list.html",
        {
            "tasks": sorted(tasks, key=lambda x: x["name"]),
            "running": False,
            "nbar": "scheduleable_tasks",
        },
    )

@check_app_status
@require_safe
def running_tasks(request):
    def add_tasks(ret, task_dict, status, scheduled=False):
        # Celery sometimes randomly fails (network glitches?)
        # Nested call chain is too complex to understand / file a bug for
        if not task_dict:
            return None
        for worker in task_dict:
            for task in task_dict[worker]:
                if scheduled:
                    # There's another level of nesting for scheduled tasks.
                    task = task["request"]

                task["status"] = status
                if task.get("time_start"):
                    task["time_start"] = timezone.datetime.utcfromtimestamp(task["time_start"])
                ret.append(task)
        # Return None to implicitly indicate ret (list of tasks) was mutated

    inspect = inspect_celery()
    tasks = []  # A list of task dicts
    add_tasks(tasks, inspect.active(), "running")
    add_tasks(tasks, inspect.reserved(), "pending")
    add_tasks(tasks, inspect.scheduled(), "scheduled", scheduled=True)
    # Sort the list of tasks so they don't jump around each time the display view is refreshed
    # Based on status (to match existing behavior) and name
    # Then start time for running tasks with same name
    # And args / kwargs for pending tasks with same name but no start time
    unstarted_task_datetime = timezone.datetime.utcfromtimestamp(0)
    tasks = sorted(
        tasks,
        key=lambda x: (
            x["status"],
            x["name"],
            x.get("time_start", unstarted_task_datetime),
            x["args"],
            x["kwargs"],
        ),
    )
    r = redis.Redis.from_url(settings.CELERY_BROKER_URL)
    return render(
        request,
        "task_list.html",
        {
            "cpu_queue_len": r.llen("cpu"),
            "fast_queue_len": r.llen("fast"),
            "slow_queue_len": r.llen("slow"),
            "tasks": tasks,
            "running": True,
            "nbar": "running_tasks",
        },
    )

@require_safe
def home(request: HttpRequest) -> HttpResponse:
    """Serve home page"""
    return render(request, "index.html")


#!/usr/bin/env python3

"""
This ROS node simulates a traffic light (with random initial state) and
publishes a "TrafficLightStatus" message. You can also add the topic
"traffic_light_visu" in rviz to display the traffic light.

To view the state machine, you may use "rosrun smach_viewer smach_viewer.py",
but you need to install corresponding libraries first using:

sudo apt install \
    ros-noetic-smach \
    ros-melodic-smach-ros \
    ros-noetic-smach-msgs \
    ros-noetic-smach-viewer \
    python-gtk2
"""

import rospy
import smach
from smach_ros import IntrospectionServer
import tf
import numpy as np
from visualization_msgs.msg import MarkerArray

from car_demo.msg import TrafficLightStatus
from traffic_light_simulator_visu import (
    getTrafficLightVisualization,
    TrafficLightControlReceiver,
)

# global instance of traffic light interaction
tl_control = TrafficLightControlReceiver()

# stop line coordinates: (xRight, yRight, xLeft, yLeft)
STOP_LINE = (35.6, -0.1, 29.7, -4.6)

# max distance for valid stop lines
MAX_STOP_LINE_DIST = 25

def distPoint2LineSegment(veh_x, veh_y, veh_phi, xr, yr, xl, yl):
    """
    Find the closest point from vehicle pose (veh_x, veh_y, veh_phi)
    to a line segment (xr, yr, xl, yl).
    This code is adapted from:
    https://stackoverflow.com/questions/849211/shortest-distance-between-a-point-and-a-line-segment
    """
    px = xl - xr
    py = yl - yr
    norm = px * px + py * py

    u = ((veh_x - xr) * px + (veh_y - yr) * py) / float(norm)
    u = np.clip(u, 0, 1)

    x = xr + u * px
    y = yr + u * py

    dx = x - veh_x
    dy = y - veh_y
    dist = np.hypot(dx, dy)

    # normal in driving direction
    nx, ny = py, -px
    if dx * nx + dy * ny < 0:
        dist = -dist  # we passed the stop line already

    dphi = veh_phi - np.arctan2(ny, nx)
    dphi = (dphi + np.pi) % (2 * np.pi) - np.pi  # normalize to -pi..pi

    return (dist, dphi)

# define general traffic light state handler
class TrafficLightState(smach.State):
    def __init__(
        self,
        tf_listener,
        tl_publisher,
        tlvis_publisher,
        color="GREEN",
        duration_s=10,
        rate_hz=10,
    ):
        smach.State.__init__(self, outcomes=["ellapsed", "aborted"])
        self.rate_hz = rate_hz
        self.duration_s = duration_s
        self.color = color
        self.tf_listener = tf_listener
        self.tl_publisher = tl_publisher
        self.tlvis_publisher = tlvis_publisher
        # NOTE: the order is important here (xRight, yRight, xLeft, yLeft)
        self.stopline = STOP_LINE
        self.dist2bumper = 2.75

    def execute(self, userdata):
        t_start = rospy.get_time()
        r = rospy.Rate(self.rate_hz)
        x1, y1, x2, y2 = self.stopline
        curr_cnt = tl_control.event_counter

        rospy.logout("Entering state %s." % (self.color))

        while (
            (tl_control.automatic) and (rospy.get_time() - t_start < self.duration_s)
        ) or ((not tl_control.automatic) and (tl_control.event_counter == curr_cnt)):

            # check if rospy has been terminated
            if rospy.is_shutdown():
                rospy.logout("Aborting state %s." % (self.color))
                return "aborted"

            # get vehicle position data
            try:
                (trans, rot) = self.tf_listener.lookupTransform(
                    "map", "din70000", rospy.Time(0)
                )  # get latest trafo between world and vehicle
                rpy = tf.transformations.euler_from_quaternion(rot)
                rospy.logdebug("pos: T=%s, rpy=%s" % (str(trans), str(rpy)))
            except (
                tf.LookupException,
                tf.ConnectivityException,
                tf.ExtrapolationException,
            ):
                rospy.logwarn("Could not receive position information!")
                r.sleep()
                continue

            # compute relative pose to stop line
            dx, dy = self.dist2bumper * np.cos(rpy[2]), self.dist2bumper * np.sin(
                rpy[2]
            )
            dist, dphi = distPoint2LineSegment(
                trans[0] + dx, trans[1] + dy, rpy[2], x1, y1, x2, y2
            )
            rospy.loginfo("dist=%.1f, dphi_deg=%.1f" % (dist, dphi * 180.0 / np.pi))

            # publish current traffic light status
            tl_status = TrafficLightStatus()
            state_dict = {
                "GREEN": tl_status.GREEN,
                "YELLOW": tl_status.YELLOW,
                "RED": tl_status.RED,
                "RED_YELLOW": tl_status.RED_YELLOW,
            }
            tl_status.header.stamp = rospy.get_rostime()
            tl_status.dist_m = dist
            tl_status.dphi_rad = dphi

            if (
                np.abs(tl_status.dist_m) > MAX_STOP_LINE_DIST
            ):  # traffic light too far away
                tl_status.dist_m = tl_status.INVALID
                tl_status.dphi_rad = tl_status.INVALID

            tl_status.state = state_dict[self.color]
            self.tl_publisher.publish(tl_status)

            # publish traffic light visualization
            tl_visualization = getTrafficLightVisualization(
                x1, y1, tl_status.header, self.color
            )
            self.tlvis_publisher.publish(tl_visualization)

            try:
                r.sleep()
            except rospy.exceptions.ROSInterruptException as e:
                rospy.logdebug(
                    f"Stopping {rospy.get_name()}, because of interrupt: {e}"
                )

        rospy.logout("Leaving state %s." % (self.color))

        return "ellapsed"

# main
def main():
    rospy.init_node("traffic_light_simulator")

    # For retrieving vehicle pose
    listener = tf.TransformListener()

    # For retrieving traffic light control commands
    tl_control.initialize()

    # For publishing the traffic lilght status and visualization
    status_publisher = rospy.Publisher(
        "traffic_light_status", TrafficLightStatus, queue_size=1
    )
    visu_publisher = rospy.Publisher("traffic_light_visu", MarkerArray, queue_size=1)

    # Create a SMACH state machine
    sm = smach.StateMachine(outcomes=["ellapsed", "aborted"])

    # Open the container
    with sm:
        # Add states to the container
        smach.StateMachine.add(
            "GREEN",
            TrafficLightState(
                color="GREEN",
                duration_s=5,
                tf_listener=listener,
                tl_publisher=status_publisher,
                tlvis_publisher=visu_publisher,
            ),
            transitions={"ellapsed": "YELLOW"},
        )
        smach.StateMachine.add(
            "YELLOW",
            TrafficLightState(
                color="YELLOW",
                duration_s=3,
                tf_listener=listener,
                tl_publisher=status_publisher,
                tlvis_publisher=visu_publisher,
            ),
            transitions={"ellapsed": "RED"},
        )
        smach.StateMachine.add(
            "RED",
            TrafficLightState(
                color="RED",
                duration_s=5,
                tf_listener=listener,
                tl_publisher=status_publisher,
                tlvis_publisher=visu_publisher,
            ),
            transitions={"ellapsed": "RED_YELLOW"},
        )
        smach.StateMachine.add(
            "RED_YELLOW",
            TrafficLightState(
                color="RED_YELLOW",
                duration_s=3,
                tf_listener=listener,
                tl_publisher=status_publisher,
                tlvis_publisher=visu_publisher,
            ),
            transitions={"ellapsed": "GREEN"},
        )

    # Set random initial state
    # state_list = ['GREEN', 'YELLOW', 'RED', 'RED_YELLOW']
    # sm.set_initial_state([random.choice(state_list)])
    sm.set_initial_state(["RED"])

    # For debugging and visualisation of state machine
    sis = IntrospectionServer("server_name", sm, "/SM_ROOT")
    sis.start()

    # Execute SMACH plan
    sm.execute()

    rospy.spin()
    sis.stop()

if __name__ == "__main__":
    main()


# -*- coding: utf-8 -*-
"""
Created on Fri Sep 10 16:48:16 2021

@author: dript
"""

'''
This file will be triggered to pull API data and Google Sheet data
and check if any new date there or not. Then merge new date if
available.

'''

import pandas as pd
from .api_data_parser import getApiData
from .sheet_data_parser import getSheetData
from datetime import datetime
import logging

class Updater():
    def __init__(self, config_path):
        self.config = config_path

    def apiData(self):
        """This pull the API data from https://data.covid19bharat.org/v4/min/timeseries.min.json

        Returns:
            dict: Gives a Dict in following format
            '{
              "confirmed":DataFrame,
              "recovered":DataFrame,
              "deceased":DataFrame
            }'
        """
        data = getApiData()
        return data()

    def sheetData(self):
        """This pull the Google Sheet data

        Returns:
            dict: Gives a Dict in following format
            '{
              "confirmed":DataFrame,
              "recovered":DataFrame,
              "deceased":DataFrame
            }'
        """
        data, spread_obj = getSheetData().pull_data_sheet(self.config)
        return data, spread_obj

    def logger(self, massage):
        """Helper function to log any massage

        Args:
            massage (str): The massage that has to be logged
        """
        logging.basicConfig(filename="stdOut.log",
                            format='%(asctime)s %(message)s',
                            filemode='a',
                            level=logging.INFO)

        logging.info(massage)

    def Validator(self, new_frame, key):
        """Validate the data whether current date values
            are greater than previous data values or not

        Args:
            new_frame (DataFrame): The new data with new dates
            key (str): A value among ["confirmed","deceased","recovered"]

        Returns:
            bool: If any new date value less than previous data returns False.
        """
        new_frame = new_frame.set_index("CODE")
        new_frame = new_frame.astype(int)

        def getName(d1, d2):
            tally = new_frame[d1] <= new_frame[d2]
            name = tally[tally == False].index[0]
            return name
        haserror = False
        col_list = new_frame.columns
        for idx, val in enumerate(col_list):
            if idx > 0:
                if (new_frame[col_list[idx - 1]] <= new_frame[col_list[idx]]).all() != True:
                    haserror = True
                    print(f"Data integrity error found for {key} check 'stdOut.log' file.")
                    self.logger(
                        f"Data Integrity Error b/w {col_list[idx - 1]} and {col_list[idx]} at {getName(col_list[idx - 1],col_list[idx])} for updating {key}")
        if haserror == True:
            return True    # Toggle this to False if don't want to update when value is less than previous day
        return True

    def prepare_for_merge(self, api_data, sheet_data, key):
        """Prepare the new data for merging with old data.

        Args:
            api_data (DataFrame): The data pull from API
            sheet_data (DataFrame): The data pulled from Google Sheet
            key (str): A value among ["confirmed","deceased","recovered"]

        Raises:
            Exception: If last updated date is not specified in API data then
            it raises an exception.

        Returns:
            DataFrame: If new date is available the new merged data will be returned.
            Otherwise old data is returned.
        """
        sheet_cols_last = sheet_data.columns[-1]
        sheet_cols_last = datetime.strptime(sheet_cols_last, "%m/%d/%Y")
        #print(api_data.columns)
        if sheet_cols_last in api_data.columns:
            idx = list(api_data.columns).index(sheet_cols_last)
            if len(list(api_data.columns)[idx:]) > 1:
                new_col = api_data.iloc[:, idx+1:].astype(int)
                new_col.columns = [i.strftime("%m/%d/%Y")
                                   for i in new_col.columns]
                new_col["CODE"] = api_data["CODE"]
                mer = pd.merge(sheet_data[["CODE",sheet_data.columns[-1]]], new_col, on="CODE", how='left')
                if self.Validator(mer, key):
                    mer = pd.merge(sheet_data, new_col, on="CODE", how='left')
                    return mer.dropna(axis=0)
                else:
                    return sheet_data
            else:
                return sheet_data
        else:
            raise Exception("Something Wrong Happended!!")

    def __call__(self):
        print("Fetching API data....")
        api = self.apiData()
        print("Fetching API data done.")
        print("Fetching Google Sheet data....")
        sheet, objs = self.sheetData()
        print("Fetching Google Sheet data done.")

        print("Performing Update of Google Sheet...")
        res_dict = {}
        for api_data, sheet_data in zip(api, sheet):
            final_data = self.prepare_for_merge(
                api[api_data], sheet[sheet_data], key=api_data)
            if final_data.shape[1] > sheet[sheet_data].shape[1]:
                res_dict[api_data] = final_data
                obj = objs[api_data]
                print(f"Updating {api_data} Google Sheet...")
                obj.df_to_sheet(final_data, index=False, replace=True)
            else:
                print(
                    f"Updating {api_data} Google Sheet is not required(already up-to-date)...")
            res_dict[api_data] = final_data
        print("Google Sheet Updating Done..")
        return res_dict


# -*- coding: utf-8 -*-
import csv
import nltk
import pdb
import pickle
from nltk.stem import WordNetLemmatizer

def addBigram(cntBigram,strBigram):
    if strBigram in cntBigram:
        cntBigram[strBigram]=cntBigram[strBigram]+1
    else:
        cntBigram[strBigram]=1
    return cntBigram

lemmatizer = WordNetLemmatizer()

# WRITE FILE
with open("final.txt","w",encoding='utf_8') as f: #open an output file
    # READ CSV FILE for sentences with wrong collocation (Wci) error tag
    filename = "wci.csv"
    #dictWrgVbNn={}
    with open(filename,'r', newline='') as csvfile:
        csvreader = csv.reader(csvfile,delimiter='\t')
        for line in csvreader:
            p1=int(line[2]) # the start position of the marked error
            p2=int(line[3]) # the end position of the marked error
            sentence=line[5]
            # split the sentence into tokens and add pos tags
            words=sentence.split(' ')
            tagged_words = nltk.pos_tag(words)
            # find Verb-Noun Bigram Wrong collocation
            flagVerb=False
            flagNoun=False
            for word_pos in tagged_words[p1:p2]:
                if word_pos[1].startswith('V'): # if there is a Verb within the marked error
                    lemma0=lemmatizer.lemmatize(word_pos[0].lower(),'v')
                    word0=word_pos[0].lower()
                    if lemma0 == 'be': # exclude 'be'
                        lemma0=[]
                        word0=[]
                        flagVerb=False
                        #pdb.set_trace()
                    else:
                        flagVerb=True
                elif (flagVerb and word_pos[0].isalpha() and word_pos[1].startswith('N')): # find the Noun after the Verb
                    flagNoun=True
                    lemma1=lemmatizer.lemmatize(word_pos[0].lower(),'n')
                    word1=word_pos[0].lower()
                    break

            if flagVerb:
                if flagNoun: # Have found a Noun in the marked error
                    #dictWrgVbNn=addBigram(dictWrgVbNn,lemma0+' '+lemma1)
                    #pdb.set_trace()
                    f.write(word0+' '+word1+'\n')
                else: # find a Noun in the following words until reach the end of sentence
                    for word_pos in tagged_words[p2:]:
                        if (word_pos[0].isalpha() and word_pos[1].startswith('N')):
                            flagNoun=True
                            lemma1=lemmatizer.lemmatize(word_pos[0].lower(),'n')
                            word1=word_pos[0].lower()
                            #dictWrgVbNn=addBigram(dictWrgVbNn,lemma0+' '+lemma1)
                            #pdb.set_trace()
                            f.write(word0+' '+word1+'\n')
                            break

# WRITE FILE
##f = open("final.txt","w",encoding='utf_8') #open an output file
##for ngram in dictWrgVbNn.keys(): #iterate over each ngram
##    words=ngram.split(' ')
##    for word in words:
##       #pdb.set_trace()
##       f.write(word)
##       f.write("\t")
##    f.write(str(dictWrgVbNn[ngram])) #then print the score
##    f.write("\n") #print a new line
##f.close() #close the output file
##
##    # open a file to store the data
##file = open('dictWrgVbNn', 'wb')
##data=dictWrgVbNn
##pickle.dump(data, file)
##file.close()


import os
import sys

from PyQt5 import QtCore, QtWidgets, QtWebEngineWidgets, uic
from PIL import Image, ImageTk
import tkinter as tk
import subprocess
options = ["Web", "Local" , "QnA"]

def on_button(option):
	print(option)
	if option == "Local":
		query = searchBox.get()
		commandList = []
	#if tokens[0] in ['file','document']:

		command = "sudo find /home -type f -name '" + query + "'"
		ans = subprocess.check_output(command, shell=True);
		command = str(ans,'utf-8').rstrip()
		commands = command.split("\n")
		for i in commands:
			commandList.append(i)
		print("in here")
		#os.system(command)
		#if tokens[0] in ['directory','folder']:
		command = "sudo find /home -type d -name '" + query + "'"
		#os.system(command)
		ans = subprocess.check_output(command, shell=True);
		#print(ans)
		command = str(ans,'utf-8').rstrip()
		commands = command.split("\n")
		for i in commands:
			commandList.append(i)
		#print(commands)
		for command in commandList:
			print(command)
			if len(command) > 0:
				os.system("gnome-open " + command)
		pass
		"""GAURAV FUNCTION"""

	elif option == "Web":
		query = searchBox.get().replace(" ", "+")
		app = QtWidgets.QApplication(sys.argv)
		path_ui = os.path.join(os.path.dirname(__file__), "test.ui")
		window = uic.loadUi(path_ui)
		window.widget.load(QtCore.QUrl("https://www.duckduckgo.com/?q="+ query ))
		window.show()
		app.exec_()
		window.destroy()
		app.destroy()

	elif option == "QnA":
		pass
		"""Shrinivas FUNCTION"""

def on_search():
	print(searchBox.get())

master = tk.Tk()
master.title('Sigma Σ')
master.geometry("%dx%d" % (1000, 500))

b= tk.Button(master,command = on_search)
#img = Image.open("sigma1.png")
photo= tk.PhotoImage(file="sigma1.png")
b.config(width="30",height="30")
b.pack()

searchBox = tk.Entry(master, textvariable = "Type here to search! ",width = 60)
searchBox.pack()

for i,option in enumerate(options):
	B= tk.Button(master, text = option, command = lambda option=option:on_button(option))
	B.pack()
master.mainloop()


from torch import nn

leaf_types = (
    nn.Conv1d,
    nn.Conv2d,
    nn.Conv3d,
    nn.Linear,
    nn.Identity,

    nn.PReLU,
    nn.ReLU,
    nn.ReLU6,
    nn.RReLU,
    nn.SELU,
    nn.CELU,
    nn.GELU,
    nn.Sigmoid,
    nn.SiLU,
    nn.Mish,
    nn.Threshold,
    nn.GLU,
    nn.Hardsigmoid,
    nn.Hardtanh,
    nn.Hardswish,
    nn.LeakyReLU,
    nn.LogSigmoid,

    nn.MaxPool1d,
    nn.MaxPool2d,
    nn.MaxPool3d,
    nn.AdaptiveMaxPool1d,
    nn.AdaptiveMaxPool2d,
    nn.AdaptiveMaxPool3d,

    nn.AvgPool1d,
    nn.AvgPool2d,
    nn.AvgPool3d,
    nn.AdaptiveAvgPool1d,
    nn.AdaptiveAvgPool2d,
    nn.AdaptiveAvgPool3d,

    nn.BatchNorm1d,
    nn.BatchNorm2d,
    nn.BatchNorm3d,
    nn.GroupNorm,
    nn.SyncBatchNorm,
    nn.LayerNorm,

    nn.Dropout,
    nn.Embedding,
    nn.Softmax,
    nn.Softmax2d,
    nn.LogSoftmax,
)

def get_other_info(module):
    # If a handler function exists for the module's type, call it
    extra_rep = ''
    if isinstance(module, leaf_types):
        extra_rep = module.extra_repr()
        if extra_rep:
            return '(' + extra_rep + ')'

    # If no specific handler, return an empty string
    return extra_rep


from django.http import HttpResponse
from django.shortcuts import render
import operator

def home(request):
    return render(request,'home.html',{'hi': '12' })
def count(request):
    fulltext = request.GET['fulltext']
    print(fulltext)
    wordlist = fulltext.split()

    worddic = {}
    for word in wordlist:
        if word in worddic:
            worddic[word] += 1
        else:
            worddic[word] = 1
    maxv = 0
    for word in worddic:
        if maxv < worddic[word]:
            maxv = worddic[word]
    sortedic = sorted(worddic.items(),key=operator.itemgetter(1), reverse = True)
    return render(request,'count.html',{'fulltext':fulltext,'count':len(wordlist),'sortedic':sortedic,'max':maxv})