#importing required libraries
import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
import seaborn as sns
from sklearn import linear_model, metrics

#reading the required dataset
wtloss=pd.read_csv('wtloss.csv')
wtloss

#describing the dataset
wtloss.describe()

#info of the dataset to know if there are any null values and get the count of elements
wtloss.info()

wtloss['Exercise']=wtloss['Exercise'].replace(0,10) #replacing the values of 0 with default value 10 in exercises

wtloss['Exercise'][wtloss['Exercise'] < 0] =np.nan #making the neg values to nan values

wtloss['Exercise']=wtloss['Exercise'].fillna(wtloss['Exercise'].mean()) #replacing the neg values with mean of the coloumn

wtloss['Weight'].isnull() #nan values in weight coloumn are dropped

wtloss=wtloss.dropna() #drops the rows with null values
wtloss['Exercise']=wtloss['Exercise'].astype('int64')

wtloss

cor=wtloss.corr() #corelation of all the variable for given data set
cor

sns.heatmap(cor) #heatmap of corelation between all the variables

#linear regression
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error
x=wtloss['Exercise']
y=wtloss['Weight']
train_x,test_x,train_y,test_y=train_test_split(x,y,test_size=.2)
model=linear_model.LinearRegression()
train_x=np.array(train_x).reshape(-1,1) #reshaping the 1d arrays to 2d array
train_y=np.array(train_y).reshape(-1,1)#reshaping 1d to 2d

model.fit(train_x,train_y)

pred_y=model.predict(train_x) #model predicted values
pred_y

#multivariate regression
mul_x=wtloss[['Exercise','Days']]
y=wtloss['Weight']
mul_y=np.array(y).reshape(-1,1)
mul_model=linear_model.LinearRegression()
mul_x,mul_test_x,mul_y,mul_test_y=train_test_split(mul_x,mul_y,train_size=.8)

mul_model.fit(mul_x,mul_y)

predmul_y=mul_model.predict(mul_x) #multivariate model predicted values
predmul_y

model.score(train_x,train_y) #linear model score

mul_model.score(mul_x,mul_y)# multivariate model score

mean_squared_error(mul_y,predmul_y) #multivariate model mean squared error

mean_squared_error(train_y,pred_y) #univariant model mean squared error

#the multivariate model is bettr than univariate model since it has greater score and lesser mean squared error


import os
import json
import boto3

def get_details(event, context):
    """
    Returns all project details from a DynamoDB table
    """

    try:
        # query the dynamo table
        dynamo_client = boto3.client("dynamodb")
        table_name = os.getenv("PROJECT_TABLE", None)
        items = dynamo_client.scan(TableName=table_name)
        items = flatten_result_set(items.get("Items"))

        # return the items as a json object
        response = json.dumps(
            {
                "items": items,
                "statusCode": 200,
            }
        )

    # return an error message as a JSON object if there is an error
    except:
        response = json.dumps({"message": "An error has occurred.", "statusCode": 500})

    return response

def flatten_result_set(result_set):
    """Flatten a dynamo resultset for returning as part of a HTTP response.
    Args:
        result_set: The result set of items returned from dynamo DB.
    Returns:
        flat_result_set: The result set flattened to remove any dynamo specific type information.
    """
    flat_result_set = []

    for row in result_set:
        flattened_row = {}

        for key in row.keys():
            value = row.get(key)

            data = list(value.values())[0]

            flattened_row[key] = data

        flat_result_set.append(flattened_row)

    return flat_result_set


import torch
import torch.nn as nn
import numpy as np
import MinkowskiEngine as ME

def make_pointwise_mlp_sparse(dims, doLastRelu=False):
    """
    Make poinwise MLP layers based on MinkowskiEngine
    """
    layers = []
    for i in range(len(dims) - 1):
        layers.append(
            ME.MinkowskiLinear(dims[i], dims[i + 1], bias=True)
        )
        if i != len(dims) - 2 or doLastRelu:
            layers.append(ME.MinkowskiReLU(inplace=True))
    return torch.nn.Sequential(*layers)

class MlpDecoderSparse(nn.Module):
    """
    MLP decoder implemented with MinkowskiEngine
    """

    def __init__(self, net_config, **kwargs):
        super(MlpDecoderSparse, self).__init__()
        self.num_points = net_config['num_points']
        dims = net_config['dims']
        self.mlp = make_pointwise_mlp_sparse(dims + [3 * self.num_points], doLastRelu=False) # the MLP layers

    def forward(self, x):
        out = self.mlp(x) # BatchSize X PointNum X 3
        return out


#!/usr/bin/python

import base64
from base64 import b64decode as b64dec
import binascii
import StringIO

def xor(a,b):
    if len(a) > len(b):
        return "".join([ chr(ord(x)^ord(y)) for (x,y) in zip(a[:len(b)], b)])
    else:
        return "".join([ chr(ord(x)^ord(y)) for (x,y) in zip(a, b[:len(b)])])

class PaddingException(Exception):
    pass

class PKCS7Encoder(object):
    '''
    RFC 2315: PKCS#7 page 21
    Some content-encryption algorithms assume the
    input length is a multiple of k octets, where k > 1, and
    let the application define a method for handling inputs
    whose lengths are not a multiple of k octets. For such
    algorithms, the method shall be to pad the input at the
    trailing end with k - (l mod k) octets all having value k -
    (l mod k), where l is the length of the input. In other
    words, the input is padded at the trailing end with one of
    the following strings:

                     01 -- if l mod k = k-1
            02 02 -- if l mod k = k-2
                        .
                        .
                        .
          k k ... k k -- if l mod k = 0

    The padding can be removed unambiguously since all input is
    padded and no padding string is a suffix of another. This
    padding method is well-defined if and only if k < 256;
    methods for larger k are an open issue for further study.
    '''
    def __init__(self, k=16):
        self.k = k

    ## @param text The padded text for which the padding is to be removed.
    # @exception ValueError Raised when the input padding is missing or corrupt.
    def decode(self, text):
        '''
        Remove the PKCS#7 padding from a text string
        '''
        nl = len(text)
        val = int(binascii.hexlify(text[-1]), 16)
        if val > self.k or val > nl:
            raise PaddingException()
        if text[-val:] != chr(val)*val:
            raise PaddingException()

        l = nl - val
        return text[:l]

    ## @param text The text to encode.
    def encode(self, text):
        '''
        Pad an input string according to PKCS#7
        '''
        l = len(text)
        output = StringIO.StringIO()
        val = self.k - (l % self.k)
        for _ in xrange(val):
            output.write('%02x' % val)
        return text + binascii.unhexlify(output.getvalue())

from Crypto import Random
from Crypto.Cipher import AES
import time

padder = PKCS7Encoder()
prng   = Random.new()

# Random 256 bit AES key
key = prng.read(32)

def encrypt(key, plaintext):
    iv = prng.read(16)
    padded_text = padder.encode(plaintext)
    cipherbytes = AES.new(key, AES.MODE_CBC, iv).encrypt(padded_text)
    return iv + cipherbytes

def decrypt(key, ciphertext):
    iv = ciphertext[:16]
    cipherbytes = ciphertext[16:]
    padded_text = AES.new(key, AES.MODE_CBC,iv).decrypt(ciphertext[16:])
    return padder.decode(padded_text)

def simple_padding_oracle(ciphertext):
    try:
        decrypt(key, ciphertext)
        return False
    except PaddingException as e:
        return True

def guess_byte(ciphertext, known, po):
    pad_size = len(known) + 1
    pad = chr(pad_size)*(pad_size-1)
    prefix = chr(0)*(16-pad_size)
    suffix = xor(pad, known) + chr(0)*16
    for b in xrange(1, 256):
        if not po(xor(prefix + chr(b ^ pad_size) + suffix, ciphertext)):
            return chr(b)

def guess_block(ciphertext, known, po):
    for x in xrange(16 - len(known)):
        known = guess_byte(ciphertext, known, po) + known
    return known

def guess_pad(ciphertext, po):
    pad = 16
    for i in xrange(16):
        if po(xor(ciphertext, chr(0)*i + chr(1) + chr(0)*(16 + 16 - i - 1))):
            pad = 16 - i
            break
        else:
            pass
    return chr(pad)*pad

def po_decrypt(ciphertext, po):
    res = guess_block(ciphertext[-32:], guess_pad(ciphertext[-32:], po), po)
    ciphertext = ciphertext[:-16]
    while len(ciphertext) > 16:
        res = guess_block(ciphertext[-32:], '', po) + res
        ciphertext = ciphertext[:-16]
    return padder.decode(res)

import time
def time_decrypts(n):
    c = encrypt(key, 'jibberish')
    start = time.time()
    for x in xrange(n):
        nop = decrypt(key, c)
    return time.time() - start

import suds
import suds_requests

po_service = suds.client.Client('https://ardoric.outsystemscloud.com/PaddingOracle/LoginService.asmx?WSDL', transport=suds_requests.RequestsTransport()).service

def ws_padding_oracle(ciphertext):
   try:
       po_service.Login('some_user', base64.b64encode(ciphertext))
       return False
   except suds.WebFault as fault:
       return 'Padding' in fault.message

# Go to https://ardoric.outsystemscloud.com/PaddingOracle/
# Encrypt a string there
# Run po_decrypt(b64dec('<your string>'), ws_padding_oracle) to check how long it takes to decrypt.
# All the Login method does is decrypt the string and let the error propagate upwards
# Takes 40s to decrypt a 3 letter message, 360 to decrypt a two block message


from collections import deque
import sys

input = sys.stdin.readline

T = int(input())

dx = [-1,1,0,0]
dy = [0,0,-1,1]

def bfs(x,y):
    q = deque()
    q.append((x,y))
    if visited[x][y] == True:
        return 0
    visited[x][y] = True
    while q:
        x,y = q.popleft()
        if graph[x][y] == 0:
            continue
        for i in range(4):
            nx = x + dx[i]
            ny = y + dy[i]
            if nx < 0 or nx >= N or ny < 0 or ny >= M:
                continue
            if graph[nx][ny] == 1 and not visited[nx][ny]:
                visited[nx][ny] = True
                q.append((nx,ny))
    return 1

for i in range(T):
    M, N, K = map(int,input().split())
    graph = [[0] * M for _ in range(N)]
    visited = [[False] * M for _ in range(N)]

    for j in range(K):
        a,b = map(int, input().split())
        graph[b][a] = 1

    result = 0

    for j in range(N):
        for k in range(M):
            if graph[j][k] == 1:
                result += bfs(j,k)

    print(result)


import json
from pprint import pprint
from purpleair.sensor import Sensor
from datetime import datetime
import pandas as pd
import plotly.graph_objects as go

SENSOR_METADATA_PATH = "/home/sgrady/dev/sensor_data/purpleair_visualization/data/sensor_metadata.json"

def get_sensor_id_by_name(sensor_name: str):
    with open(SENSOR_METADATA_PATH, "r") as metadata:
        sensor_metadata = json.load(metadata)
        sensor_id = sensor_metadata[sensor_name]["id"]
    return sensor_id

def get_sensor_historical_data(
    sensor: Sensor,
    channel: str,
    field: str,
    weeks: int,
) -> pd.DataFrame:
    channels = {
        "parent": sensor.parent,
        "child": sensor.child,
    }
    historical_dataframe = channels[channel].get_historical(
        weeks_to_get=weeks,
        thingspeak_field=field,
    )
    return historical_dataframe


# Author: 海子
# Create Time: 2021/8/19
# FileName: BRIEF
# Descriptions:  特征生成流程：特征点快速提取法CenSurf（STAR）+BRIEF特征描述+汉明距离匹配
import numpy as np
import cv2 as cv
from matplotlib import pyplot as plt

img = cv.imread('img/01.jpeg', 0)

# 初始化Star检测器
star = cv.xfeatures2d.StarDetector_create()

# 初始化BRIEF提取器
brief = cv.xfeatures2d.BriefDescriptorExtractor_create()
t1 = cv.getTickCount()
# 找到STAR的关键点
kp = star.detect(img, None)
img2 = cv.drawKeypoints(img, kp, None, color=(255, 0, 0))
t2 = cv.getTickCount()
time = (t2 - t1)/cv.getTickFrequency()
# 打印所有默认参数
print("SURF-->time=%s ms" % (time*1000))
cv.imshow("kp", img2)
# 计算BRIEF的描述符
kp, des = brief.compute(img, kp)  # 该des可以是128、256或512。OpenCV支持所有这些，但默认情况下将是为单位表示，因此值将为16、32和64）。因此，一旦获得此信息，就可以些描述符。

print(brief.descriptorSize())
print(des.shape)
cv.waitKey()
cv.destroyAllWindows()

特征检测与提取/特征描述/BRIEF.py


# imports
from imports import *
from selenium import webdriver
from webdriver_manager.chrome import ChromeDriverManager

chrome_options = Options()
chrome_options.add_argument("--headless")
driver = webdriver.Chrome(
    ChromeDriverManager().install(), options=chrome_options)

# Color and fonts
dcolor = '#111111'
frameColor = '#222222'
widFont = 'helvetica bold'

# Basic buildup
root = tk.Tk()
root.geometry('800x600')
root.configure(background=dcolor)
root.resizable(0, 0)
root.title('Pg Searcher')

# Menu Bar
def onOpen():
    global chosendir
    chosendir = filedialog.askopenfilename(
        initialdir="/", title="Open file", filetypes=(("Text files", "*.txt"), ("All files", "*.*")))

def onSave():
    global savedfiles
    savedfiles = filedialog.asksaveasfilename(initialdir="/", title="Save as",
                                              filetypes=(("Text files", "*.txt"), ("All files", "*.*")))

menubar = tk.Menu()
filemenu = tk.Menu(menubar, tearoff=0)
filemenuHelp = tk.Menu(menubar, tearoff=0)
filemenu.add_command(label="Open", command=onOpen)
filemenu.add_command(label="Save", command=onSave)
filemenu.add_command(label="Exit", command=root.quit)
menubar.add_cascade(label="File", menu=filemenu)
menubar.add_cascade(label="Help", menu=filemenuHelp)
filemenuHelp.add_command(label="Help")
filemenuHelp.add_command(label="About")
root.config(menu=menubar)

# Frame, labels, entries
vert1 = Frame(root, bg=frameColor, height=400, width=500)
vert1.place(relx=0.50, rely=0.50, anchor='center')
vert1.grid_columnconfigure((0), weight=1)
vert1.grid_rowconfigure((0, 1, 2, 3, 4), weight=1)
vert1.grid_propagate(False)

Label1 = Label(vert1, text="Your query", height=2, fg='white',
               bg=frameColor, font=(widFont, 15))
Label2 = Label(vert1, text="The amount of pages you want to search",
               height=2, fg='white', bg=frameColor, font=(widFont, 15))

Entry1 = Entry(vert1, width=40, font=(widFont, 15))
Entry2 = Entry(vert1, width=40, font=(widFont, 15))

Label1.grid(row=0, column=0, padx=5)
Entry1.grid(row=1, column=0, padx=5)
Label2.grid(row=2, column=0, padx=5)
Entry2.grid(row=3, column=0, padx=5)

def getInput():
    global inp1, inp2
    inp1 = Entry1.get()
    inp2 = int(Entry2.get())

    for page in range(1, inp2):
        url = "http://www.google.com/search?q=" + \
            inp1 + "&start=" + str((inp2 - 1) * 10)
        driver.get(url)
        soup = BeautifulSoup(driver.page_source, 'html.parser')
        search = soup.find_all('div', class_="yuRUbf")
        for h in search:
            print((h.a.get('href')), file=open(chosendir, 'a'))

# Button
submitButton = Button(vert1, width=10, text='Submit',
                      font=(widFont, 15), command=getInput)
submitButton.grid(row=4, column=0, pady=5)

# Mainloop
root.mainloop()


""" function to visualize results from prediction"""
import copy
import pandas as pd
from math import isnan
import plotly.graph_objects as go

def visualize_predict(suffix = "", code = "", tick = ""):
    """
    function to visualize the results from a prediction
    Input :
    - suffix : string to identify the csv file to load
    """
    # load data
    df = pd.read_csv("df_eval_"+suffix+code+".csv")
    # to normalize prices and balance
    idx_init = df['balance'].notna().idxmax()
    init_Close = df['Close'].values[idx_init]
    init_balance = next(x for x in df['balance'].values if not isnan(x))
    # infor for sell
    dfsell = copy.deepcopy(df)
    dfsell = dfsell[dfsell["sell"] > 0]
    xsell = dfsell["Date"]
    ysell = dfsell["Close"]
    sizesell = dfsell["volume_order"].values*20
    # info for buy
    dfbuy = copy.deepcopy(df)
    dfbuy = dfbuy[dfbuy["buy"] > 0]
    xbuy = dfbuy["Date"]
    ybuy = dfbuy["Close"]
    sizebuy = dfbuy["volume_order"].values*20
    # Create figure
    fig = go.Figure(
        data=[go.Candlestick(x=df['Date'],
                    open=(df['Open']/init_Close-1)*100,
                    high=(df['High']/init_Close-1)*100,
                    low=(df['Low']/init_Close-1)*100,
                    close=(df['Close']/init_Close-1)*100,
                    name=tick,
                    increasing_line_color= '#BFC253', decreasing_line_color= '#53C2BA'),
                go.Scatter(x=xsell,
                    y=(ysell/init_Close-1)*100,
                    mode='markers',
                    name="buy",
                    marker=dict(
                        color="red",
                        size=sizesell,
                        opacity = 0.5,
                        sizemin=0)),
                go.Scatter(x=xbuy,
                    y=(ybuy/init_Close-1)*100,
                    mode='markers',
                    name="sell",
                    marker=dict(
                        color="green",
                        size=sizebuy,
                        opacity = 0.5,
                        sizemin=0)),
                go.Scatter(x=df['Date'],
                    y=(df['balance']/init_balance-1)*100,
                    mode='lines',
                    line=dict(
                    color="#0205C7",
                    width=3),
                    opacity=0.5,
                    name="account balance")
            ])
    fig.update_xaxes(rangeslider_visible=False)
    fig.update_layout(title=suffix+" "+code,
                    xaxis_title="Date",
                    yaxis_title="Relative Scale %",
                    plot_bgcolor='#F5F5F5',
                    paper_bgcolor='rgba(0, 0, 0, 0)',
                    font=dict(
                    size=22,  # Set the font size here
                    color="Black"
                    ))
    fig.update_yaxes(showgrid=True,
                    gridwidth=1,
                    gridcolor='#CACBCB')
    fig.update_xaxes(showgrid=False)

    fig.show()

#visualize_predict(suffix = "train", tick = "AAPL")
#visualize_predict(suffix = "valid", tick = "AAPL")


# ===============[ IMPORTS ]===============
import re
import multiprocessing
import nltk
import requests
import pandas as pd
import matplotlib.pyplot as plt
import sklearn.manifold as sk
from gensim.models.word2vec import Word2Vec

# ===============[ LOADING TEXT CORPUS ]===============
nltk.download('punkt')
nltk.download('stopwords')

# ===============[ TEXT PRE-PROCESSING ]===============
def sentence_to_wordlist(raw):
    clean = re.sub('[^a-zA-Z]', ' ', raw).lower()
    words = clean.split()
    return words

# ===============[ DATASET PRE-PROCESSING ]===============
filepath = 'http://www.gutenberg.org/files/33224/33224-0.txt'
corpus_raw = requests.get(filepath).text

tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')
raw_sentences = tokenizer.tokenize(corpus_raw)

sentences = []
for raw_sentence in raw_sentences:
    if len(raw_sentence) > 0:
        sentences.append(sentence_to_wordlist(raw_sentence))

# ===============[ HYPERPARAMETERS TUNING ]===============
num_features = 300
num_workers = multiprocessing.cpu_count()

seed = 1
context_size = 7
downsampling = 1e-3
min_word_count = 3

# ===============[ MODEL BUILDING ]===============
model2vec = Word2Vec(
    size=num_features,
    workers=num_workers,
    sg=1,
    seed=seed,
    window=context_size,
    sample=downsampling,
    min_count=min_word_count,
)

model2vec.build_vocab(sentences)

# ===============[ TRAINING ]===============
model2vec.train(sentences, total_examples=model2vec.corpus_count, epochs=100)
model2vec.save('vectors/trained_model.w2v')

# ===============[ DIMENSIONALITY REDUCTION ]===============
tsne = sk.TSNE(n_components=2, random_state=0)
all_word_vectors_matrix_2d = tsne.fit_transform(model2vec.wv.vectors)

# ===============[ CLUSTER VISUALIZATION ]===============
word_vectors_list = [(word, all_word_vectors_matrix_2d[model2vec.wv.vocab[word].index]) for word in model2vec.wv.vocab]
word_coordinates_list = [(word, coords[0], coords[1]) for word, coords in word_vectors_list]

points = pd.DataFrame(
    word_coordinates_list,
    columns=['word', 'x', 'y']
)

plt.scatter(points['x'], points['y'])
plt.show()


'''
Given target name, connects to the SQLite database and pulls frequencies necessary for ranking.
The main function, get_rankings, returns a database with disease name and rankings.
Requires ranking module in same dir.
'''

import sqlite3
import pandas as pd
import sqlalchemy
from sqlalchemy import create_engine

import ranking

from fuzzywuzzy import fuzz
from fuzzywuzzy import process

import ast
from collections import Counter

#Lookup functions: names and ids

def get_disease_name(disease_id, engine):
    with engine.connect() as con:
        rs = con.execute(f"""
        SELECT Disease
        FROM malacard
        WHERE Disease_ID = {disease_id}
        """)
        counts = rs.fetchall()
    name = counts[0][0]

    return name

def find_similar(targ_abbr, engine):
    '''
    Gets 5 top matches to the term from target list.

    Arguments:
        targ_abbr: target abbreviation, search term
        engine: connection to SQLite database

    Returns:
        clean_matches: list of 5 top matches to the target
    '''
    with engine.connect() as con:
        rs = con.execute("""
        SELECT targ_abbr
        FROM targets
        """)
        counts = rs.fetchall()
        targ_abbrs = [i[0] for i in counts]
        matches = process.extract(targ_abbr, targ_abbrs, limit = 5)
        #print(targ_abbrs)
        #matches = [i for i in targ_abbrs if fuzz.ratio(targ_name, i) >40]

        clean_matches = [i[0] for i in matches]

        return clean_matches

def get_dis_year(dis_id, engine):
    '''
    Get number of publications per year for a given disease_id.

    Arguments:
        dis_id: disease_id
        engine: connection to SQLite database

    Returns:
        dataframe with pmid, disease_id, year
    '''
    with engine.connect() as con:
        df = pd.read_sql_query(
                f"""
            SELECT disease_pubmed.pmid, disease_id, pubmed.year
            FROM disease_pubmed
            JOIN pubmed ON pubmed.pmid = disease_pubmed.pmid
            WHERE disease_id = {dis_id}
            """, con)
            #counts = rs.fetchall()
            #info = counts#[0]

        return df

def get_dis_id(dis_name, engine):
    '''
    Get disease id based on name.
    Arguments:
        dis_name: disease name from table. This lookup can be avoided if the whole ranking dataframe is saved
        engine: connection to SQLite
    Returns:
        info: disease id from malacard table
    '''
    with engine.connect() as con:
        try:
            rs = con.execute("""
            SELECT Disease_id
            FROM malacard
            WHERE Disease = ?
            """, (dis_name,))
            counts = rs.fetchall()
            info = counts[0][0]

            return info

        except:
            return 'NaN'

def get_target_id(target_name, engine):
    '''
    Takes the target name and returns it's id.

    Arguments:
        target_name: target name (only abbreviation for now).
        engine: sqlalchemy engine, connected to the database
    Returns:
        name: integer, target id based on DB

    '''
    with engine.connect() as con:
        try:
            rs = con.execute("""
            SELECT target_id, targ_abbr, targ_name
            FROM targets
            WHERE (targ_name = ?1 OR targ_abbr = ?2)
            """, (target_name, target_name))
            counts = rs.fetchall()
            info = counts[0]

            return info

        except:
            return 'NaN'

def get_target_name(target_id, engine):
    '''
    Takes target id and returns name abbreviation
    '''
    with engine.connect() as con:
        rs = con.execute(f"""
        SELECT targ_abbr
        FROM targets
        WHERE target_id = {target_id}
        """)
        counts = rs.fetchall()
        name = counts[0][0]
    return name

#Lookup for clinical trials

def get_pubmed_year(disease_id, engine):
    '''
    Get number of publications per year for a given disease_id.

    Arguments:
        dis_id: malacard disease_id
        engine: connection to SQLite database

    Returns:
        yr_counts: dictionary with counts of how many publications we found for given disease per year
    '''

    with engine.connect() as con:
        rs = con.execute("""
            SELECT *
            FROM pubmed_disease_frequencies
            WHERE disease_id = ?
        """,(disease_id,))
        counts = rs.fetchall()
        yrs = list(range(2009,2021))
        yr_counts = {}
        for i in yrs:
            yr_counts[i] = counts[0][yrs.index(i)+1]

    return yr_counts

def get_ct_year(dis_name, engine):
    '''
    Get number of clinical trial per year for a given disease_name.
    TODO change to id

    Arguments:
        dis_name: disease_name
        engine: connection to SQLite database

    Returns:
        counts: dictionary with counts of how many clinical trials we found for given disease per year
    '''
    with engine.connect() as con:
        with engine.connect() as con:
            rs = con.execute("""
                SELECT CT_trial_years
                FROM ct_diseases
                WHERE Disease = ?
            """,(dis_name,))
            counts = rs.fetchall()
            if counts == []:
                yr_counts = []
                return yr_counts
            else:
                info = counts[0][0]
                years = ast.literal_eval(info)
                yr_counts = dict(Counter(years))

                return yr_counts

def get_drug_info(target_id, engine):
    '''
    Takes target id and checks the DB for all the drugs that are in work related to the target.
    Based on the TTD data http://db.idrblab.net/ttd/.
    Arguments:
        target_id: int, target id based on malacard table in DB
        engine: DB connection
    Returns:
        df_sql: dataframe with drug names and their statuses
    '''

    with engine.connect() as con:
        df_sql = pd.read_sql_query(f"""
        SELECT Drug_name, Drug_status, INDICATI
        FROM drug_target_indication
        WHERE target_id = {target_id}
        """, con)

        return df_sql

def ct_forecast(dis_id, engine):
    '''
    Gets data with timeseries forecast for number of trials for a given disease.
    The forecast has been done for the full year of 2020 and 2021.
    When there was not enough data for a prediction and there is nothing in the table,
    will return a [0,0] prediction.
    '''
    with engine.connect() as con:
        rs = con.execute(f"""
            SELECT *
            FROM ct_forecast
            WHERE disease_id = {dis_id}
        """)
        counts = rs.fetchall()
        try:
            predictions = [counts[0][1], counts[0][2]]
        except:
            #When there was not enough data
            predictions = [0,0]

    return predictions

#Counting functions for ranking table

def tg_count(target_id, engine):
    '''
    Connects to a SQLite DB and counts in how many articles is the given target mentioned.
    This count is defined as n2 in metrics.

    Arguments:
        target_id: target id (as defined by ids in the "targets" table).
        engine: sqlalchemy engine, connected to the database
    Returns:
        num_tg: integer, number of articles that mention a given target.
    '''
    with engine.connect() as con:
        rs = con.execute(f"""
        SELECT COUNT(*)
        FROM target_pubmed
        WHERE target_id = {target_id}
        """)
        counts = rs.fetchall()
    num_tg = counts[0][0]

    return num_tg

def tg_dis_counts(target_id, engine):
    '''
    Connects to a SQLite DB and counts mentions of each disease together with a given target
    in the abstracts.
    This contributes to counting n0 (disease + target mentions) and n3 (disease mentions) in metrics.

    Arguments:
        target_id: target id (as defined by Malacard ids in the DB, defined in "targets" table)
        engine: sqlalchemy engine, connected to the database
    Returns:
        target_disease_counts: dictionary. Contains pairs 'disease_id' vs number of mentions with the given target
    '''

    with engine.connect() as con:
        df_sql = pd.read_sql_query(f"""
        SELECT disease_pubmed.pmid, target_id, disease_pubmed.disease_id  FROM target_pubmed
        JOIN disease_pubmed ON target_pubmed.pmid = disease_pubmed.pmid
        WHERE target_id = {target_id}

        """, con)
    con.close()
    #df_sql
    target_disease_counts = df_sql.groupby('disease_id').size().to_dict()

    return target_disease_counts

def dis_count(disease_id, engine):
    '''
    Connects to a SQLite DB and counts in how many articles is the given disease mentioned.
    This function copies the tg_count and can be reduced later.
    This count is defined as n3 in metrics.

    Arguments:
        disease_id: disease id (as defined by Malacard ids in the DB, defined in "diseases" table).
        engine: sqlalchemy engine, connected to the database
    Returns:
        num_dis: integer, number of articles that mention a given target.
    '''
    with engine.connect() as con:
        rs = con.execute(f"""
        SELECT COUNT(*)
        FROM disease_pubmed
        WHERE disease_id = {disease_id}
        """)
        counts = rs.fetchall()
    num_dis = counts[0][0]

    return num_dis

def tg_per_disease(target_disease_count, engine):
    '''
    Takes in the resulting dictionary from tg_dis_counts (disease_id vs no of mentions together with target)
    and expands by adding counts of overall articles mentioning each disease.
    The target id is not passed, it is a global variable, not used here.

    Arguments:
        target_disease_counts: dictionary, containing disease_id and count of mentions together with a target.
        engine: sqlalchemy engine, connected to the database
    Returns:
        diasease_dict: dictionary that has disease id, no of times mentioned with target, no mentioned alltogether.

    '''

    disease_dict = {}

    for key, val in target_disease_count.items():
        separate = dis_count(key, engine)
        local_dict = {'joined': val, 'separate': separate}
        disease_dict[key] = local_dict

    return disease_dict

def get_rankings(target_id, engine, fast=True, full=False):
    '''
    Overarching function that combines all the functions in this module.
    Takes target id and returns a dataframe with 4 metrics for each disease associated with target in articles.

    Arguments:
        target_id: target id (as defined by Malacard ids in the DB, defined in "targets" table)
        engine: sqlalchemy engine, connected to the database
        full: boolean.
            True: dataframe will be returned containing all columns
            False: default. Dataframe will be returned only containing columns with disease name and rankings
        fast:
            True: takes precomputed values
            Fast: checks DB and computes at the background. Tends to get slow on weak computers.

    Returns:
        df_counts: dataframe with rankings for diseases
    '''

    if fast:

        with engine.connect() as con:

            df_sql = pd.read_sql_query(f"""
            SELECT *
            FROM target_disease
            WHERE target_id = {target_id}
            """, con)
        df_sql['disease'] = df_sql.apply(lambda row: get_disease_name(row['disease_id'], engine), axis=1)
        df_sql = df_sql[['disease', 'f0', 'f1', 'f2', 'pmi']]
        df_sql = df_sql.round(4)

        return df_sql.sort_values(by='pmi', ascending = False)

    else:
        n1 = 9680305 #Number of all articles in the DB. Computing separately takes more time.
        n2 = tg_count(target_id, engine)

        target_disease = tg_dis_counts(target_id, engine)
        taget_disease_count = tg_per_disease(target_disease, engine)

        df_counts = pd.DataFrame.from_dict(taget_disease_count, orient='index')
        df_counts = df_counts.reset_index().rename(columns = {'index': 'disease_id'})

        df_counts['disease'] = df_counts.apply(lambda row: get_disease_name(row['disease_id'], engine), axis=1)
        df_counts['f0'] = df_counts.apply(lambda row: ranking.f0(n0=row['joined'], n1=n1, r=True), axis=1)
        df_counts['f1'] = df_counts.apply(lambda row: ranking.f1(n0=row['joined'], n2=n2, r=True), axis=1)
        df_counts['f2'] = df_counts.apply(lambda row: ranking.f2(n0=row['joined'], n3=row['separate'], r=True), axis=1)
        df_counts['pmi'] = df_counts.apply(lambda row: ranking.pmi(n0=row['joined'], n1=n1, n2=n2, n3=row['separate'], r=True), axis=1)

        if full:
            return df_counts.sort_values(by='pmi', ascending = False)
        else:
            return df_counts[['disease', 'f0', 'f1', 'f2', 'pmi']].sort_values(by='pmi', ascending = False)

if __name__ == '__main__':

    engine = create_engine('sqlite:///./data/20200723pubmed.db', echo=False)
    con = engine.connect()

    rankings = get_rankings(15, engine)
    print(rankings)


#!/usr/bin/python3
import boto3
import sys
import json

#Usage:
# Create a config.json file with tags:

# {
#   "tags": {
#     "Inspector": [{
#        "Key" : "PenTesting1632",
#        "Value" : "InspectorEC2Instance"
#  }]
#   }
# }

# Supply as command line argument e.g.

# tag-ec2-instances.py config.json

config_file = str(sys.argv[1])

with open(config_file, 'r') as f:
    config = json.load(f)

ec2 = boto3.resource('ec2')
mytags = config['tags']['Inspector']
instance_ids = []

def get_all_instance_ids():
   instance_ids = []

   reservations = ec2.instances.all()
   for reservation in reservations:
    #print(reservation.id)
      instance_ids.append(reservation.id)

def filter_instances(environment):
   instance_ids = []

   reservations = ec2.instances.filter(
      Filters=[{'Name': 'tag:opsworks:instance', 'Values': [environment]}])
   for reservation in reservations:
      print(reservation.id, reservation.instance_type)
      instance_ids.append(reservation.id)
   #print(instance_ids)
   tag_instances(instance_ids)

def tag_instances(instance_ids):

      for instance in instance_ids:
         ec2.create_tags(
         Resources=[instance],
         Tags= mytags
         )

filter_instances('*test*')


# -*- coding: utf-8 -*-
import scrapy
from Htqyy.items import HtqyyItem

class HtqyySpider(scrapy.Spider):
    name = 'htqyy'
    allowed_domains = ['www.htqyy.com']
    # start_urls = ['http://www.htqyy.com/']

    def start_requests(self):
        url_list = {
            "hot":"http://www.htqyy.com/top/musicList/hot?pageIndex={}&pageSize=20",
            "new":"http://www.htqyy.com/top/musicList/new?pageIndex={}&pageSize=20",
            "recommend":"http://www.htqyy.com/top/musicList/recommend?pageIndex={}&pageSize=20",
            "_1":"http://www.htqyy.com/genre/musicList/1?pageIndex={}&pageSize=20&order=hot",
            "_3":"http://www.htqyy.com/genre/musicList/3?pageIndex={}&pageSize=20&order=hot",
            "_5":"http://www.htqyy.com/genre/musicList/5?pageIndex={}&pageSize=20&order=hot",
        }

        for key,value in url_list.items():
            for url in self.page_requests(value,key):
                # print(url)
                yield scrapy.Request(url)

    def page_requests(self,url_str,type):
        pages = {
            "hot": 25,
            "new": 5,
            "recommend": 8,
            "_1": 19,
            "_3": 16,
            "_5": 8,
        }

        for index in range(pages[type]):
            yield url_str.format(index)

    def parse(self, response):

        mItem = response.xpath("//ul/li[@class='mItem']")

        for sinfo in mItem:
            item = HtqyyItem()

            item = {
                "sid":"http://f2.htqyy.com/play7/" + sinfo.xpath(".//input/@value").get() + "/mp3/3",
                "sname":sinfo.xpath(".//span[@class='title']/a/text()").get()
            }

            yield item


from wsgiref.simple_server import make_server

PORT = 3333

class Application(object):
    """creating wsgi app using class

    Args:
        object (_type_): _description_
    """
    def __call__(self,environ,response:callable):

        PATH_INFO = environ["PATH_INFO"]

        if PATH_INFO == "/hello":
            return self.hello

        status = "200 OK"
        headers = [
            ("Content-Type","text/html")
        ]
        response(
            status,
            headers
        )

        response_byte = "Hello, World !".encode()

        return [response_byte]

    def hello(self,environ,response):
        status = "200 OK"
        headers = [
            ("Content-Type","text/html")
        ]
        response(
            status,
            headers
        )

        response_byte = "Hello, World !".encode()

        return [response_byte]

    def goodbye(self,environ,response):
        status = "200 OK"
        headers = [
            ("Content-Type","text/html")
        ]
        response(
            status,
            headers
        )

        response_byte = "Good bye!".encode()

        return [response_byte]

    def notfound(self,environ,response):
        status = '404 Not Found'
        headers = [
            ("Content-Type","text/html")
        ]
        response(
            status,
            headers
        )

        response_byte = "Not found".encode()

        return [response_byte]

def app(environ,response:callable):
    # print("environ ",environ)
    status = "200 OK"
    headers = [
        ("Content-Type","text/html")
    ]
    response(
        status,
        headers
    )

    response_byte = "Hello, World !".encode()

    return [response_byte]

start_message = "server running on port {port}\nvisit link to view pages\n\n http://localhost:{port}\n\n".format(port=PORT)

if __name__ == "__main__":
    """
    you can run the server by using this to method
    """
    # try:
    #     httpd = make_server("",PORT,app)
    #     print(start_message)
    #     httpd.serve_forever()
    # except KeyboardInterrupt:
    #     print("Goodbye!")

    with make_server("",PORT,Application()) as httpd:
        print(start_message)
        httpd.serve_forever()


from django.shortcuts import render, redirect, get_object_or_404
from django.contrib import messages
from django.views.generic import ListView, DetailView
from django.utils.translation import gettext as _
from aurora.frontend.models import Post
from aurora.frontend.views import frontendView
from django.shortcuts import get_object_or_404
from aurora.backend.library import site
from django.utils.html import strip_tags

class PostLV(frontendView, ListView):
    model = Post
    section_title = _('Posts')
    template_name = 'frontend/sections/posts.html'
    fluid_width = True
    paginate_by = 10

class PostDetailView(frontendView, DetailView):
    model = Post
    section_title = _('Post')
    template_name = 'frontend/sections/post.html'
    fluid_width = True

    def get_section_title(self):
        current_object = self.get_object()
        return current_object.title

    def get_object(self):
        return get_object_or_404(self.model, slugname = self.kwargs['slugname'])

    def get_section_headers(self):
        current_object = self.get_object()
        current_url = "{0}{1}".format(site.get_current_domain(self.request), self.request.path)
        opengraph   = "<meta property='og:url' content='{0}' />".format(current_url)
        opengraph  += "<meta property='og:type' content='article' />"
        opengraph  += "<meta property='og:title' content='{0}' />".format(current_object.title)
        opengraph  += "<meta property='og:description' content='{0}' />".format(strip_tags(current_object.content[:95]))
        if current_object.thumbnail:
            thumbnail_url = current_object.thumbnail.url
        else:
            thumbnail_url = '/static/frontend/media/default-post.jpg'
        opengraph += "<meta property='og:image' content='{0}{1}' />\n".format(site.get_current_domain(self.request), thumbnail_url)
        return opengraph

    def get_context_data(self, *args, **kwargs):
        current_object = self.get_object()
        # menyimpan data meta terkait post
        if current_object.meta:
            # menyimpan data jumlah pengunjung
            if 'view_count' in current_object.meta:
                prev_count = current_object.meta['view_count']
                current_object.meta = {'view_count': prev_count + 1}
            else:
                current_object.meta = {'view_count': 1}
        else:
            current_object.meta = {'view_count': 1}
        current_object.save()
        context = super().get_context_data(*args, **kwargs)
        context['section_headers'] = self.get_section_headers()
        context['related_posts'] = self.model.objects.filter(category = current_object.category).exclude(pk = current_object.pk)[:15]
        return context


import torch
import torchvision.models
from torch import nn

class Identity(nn.Module):
    """An identity layer"""

    def __init__(self):
        super(Identity, self).__init__()

    def forward(self, x):
        return x

class ResNet(torch.nn.Module):
    """ResNet with the softmax chopped off and the batchnorm frozen"""

    def __init__(self):
        super(ResNet, self).__init__()
        self.network = torchvision.models.resnet50(pretrained=True)
        self.n_outputs = 2048

        # self.network = remove_batch_norm_from_resnet(self.network)

        # adapt number of channels
        nc = 3
        if nc != 3:
            tmp = self.network.conv1.weight.data.clone()

            self.network.conv1 = nn.Conv2d(nc, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)

            for i in range(nc):
                self.network.conv1.weight.data[:, i, :, :] = tmp[:, i % 3, :, :]

        # save memory
        del self.network.fc
        self.network.fc = Identity()

        self.freeze_bn()
        self.dropout = nn.Dropout()

    def forward(self, x):
        """Encode x into a feature vector of size n_outputs."""
        return self.dropout(self.network(x))

    def train(self, mode=True):
        """
        Override the default train() to freeze the BN parameters
        """
        super().train(mode)
        self.freeze_bn()

    def freeze_bn(self):
        for m in self.network.modules():
            if isinstance(m, nn.BatchNorm2d):
                m.eval()


from matplotlib import pyplot as plt
import json

file = open('ulamr10000_1_2.json',)
data = json.load(file)
file.close()
freqs = {}
y = []
xx = []
index = 0

# checks how frequently a number of sums shows up
for i in range(0, data[len(data)-1][0]):
    if(data[index][0] == i):
        x = str(data[index][1])
        index += 1
        if(x in freqs):
            freqs[x] = freqs.get(x) + 1
        else:
            freqs[x] = 1
    else:
        if('1' in freqs):
            freqs['1'] = freqs.get('1') + 1
        else:
            freqs['1'] = 1

for i in range(0,data[len(data)-1][0]):
    x = str(i)
    if(x in freqs):
        xx.append(i)
        y.append(freqs[x])

print(freqs)

plt.plot(xx,y)
plt.show()


import logging.config
from pathlib import Path
from IPython.core.magic import register_line_magic

# Find the project root (./../../../)
startup_error = None
project_path = Path(__file__).parents[3].resolve()

@register_line_magic
def set_display_options():
    global current_dir
    global project_path
    global context
    global catalog
    global load_context
    global InteractiveShell
    global display
    global HTML
    global pd
    global sns
    global plt

    try:
        from pathlib import Path
        from kedro.framework.context import load_context
        from IPython.core.interactiveshell import InteractiveShell
        from IPython.display import display, HTML
        import pandas as pd
        import seaborn as sns  # noqa
        import matplotlib.pyplot as plt  # noqa
        import InteractiveShell
        import warnings

        InteractiveShell.ast_node_interactivity = "all"
        warnings.filterwarnings("ignore")
        warnings.simplefilter("ignore")

        pd.set_option("display.multi_sparse", False)
        pd.set_option("display.max_rows", 500)
        pd.set_option("precision", 4)
        # pd.set_option('display.notebook_repr_html',True)
        pd.set_option("display.precision", 4)

        current_dir = Path.cwd()  # this points to 'notebooks/' folder
        proj_path = current_dir.parent  # point back to the root of the project
        context = load_context(proj_path)
        catalog = context.catalog

    except ImportError:
        logging.error("There was a problem loading the libraries ")
        raise

def isnotebook():
    try:
        shell = get_ipython().__class__.__name__
        if shell == "ZMQInteractiveShell":
            return True  # Jupyter notebook or qtconsole
        elif shell == "TerminalInteractiveShell":
            return False  # Terminal running IPython
        else:
            return False  # Other type (?)
    except NameError:
        return False

if isnotebook():
    set_display_options


from django.contrib.contenttypes.models import ContentType
from versatileimagefield.serializers import VersatileImageFieldSerializer
from rest_framework import serializers
from utils.urls import (
    split_url, valid_url_mimetype
)
from django.utils.translation import ugettext as _
from users.serializers import UserSerializer
from mezzanine.generic.models import Rating
from django_comments.models import Comment
from taggit_serializer.serializers import (
    TagListSerializerField, TaggitSerializer)
from django.conf import settings
from .models import Post

def get_post_content_type():
    return ContentType.objects.get_for_model(Post)

def _invalidate(msg):
    raise serializers.ValidationError(_(msg))

class RatingSerializer(serializers.ModelSerializer):
    class Meta:
        model = Rating
        fields = ['id', 'user', 'value', 'object_pk']
        filter_fields = ('user', 'object_pk',)
        read_only_fields = ['id', 'user']

class CommentSerializer(serializers.ModelSerializer):
    user = UserSerializer(read_only=True)

    class Meta:
        model = Comment
        fields = ['id', 'user', 'object_pk', 'comment', 'submit_date']
        filter_fields = ['user', 'object_pk']
        read_only_fields = ('user', 'submit_date')

class PostSerializer(TaggitSerializer, serializers.ModelSerializer):
    user = UserSerializer(read_only=True)
    user_rating = serializers.SerializerMethodField()
    comments_count = serializers.SerializerMethodField()
    tags = TagListSerializerField()
    # image = VersatileImageFieldSerializer(
    #     required=False, sizes='post_image')
    image = serializers.SerializerMethodField(required=False)

    class Meta:
        model = Post
        fields = [
            'id', 'title', 'link', 'link_type', 'description', 'tags',
            'publish_date', 'rating_sum', 'user', 'user_rating',
            'comments_count', 'image'
        ]
        read_only_fields = ['user', 'user_rating', 'publish_date', 'image']

    def get_image(self, obj):
        if obj.image:
            return{
                'full_size': obj.image.url,
                'medium_square_crop': obj.image.crop['280x280'].url,
                'small_square_crop': obj.image.crop['50x50'].url,
                'thumbnail': obj.image.crop['280x280'].url,
            }
        else:
            beingIndex = obj.link.index("v=")
            parameters = obj.link[beingIndex + 2:]
            endIndex = parameters.index("&") if "&" in parameters else len(parameters)
            video_id = parameters[:endIndex]
            return{
                'full_size': 'http://img.youtube.com/vi/{}/maxresdefault.jpg'.format(video_id),
                'medium_square_crop': 'http://img.youtube.com/vi/{}/hqdefault.jpg'.format(video_id),
                'small_square_crop': 'http://img.youtube.com/vi/{}/mqdefault.jpg'.format(video_id),
                'thumbnail': 'http://img.youtube.com/vi/{}/1.jpg'.format(video_id),
            }

    def get_user_rating(self, obj):
        rating = obj.rating.filter(user=self.context['request'].user)

        return RatingSerializer(
            rating.first(), many=False
        ).data if rating.exists() else None

    def get_comments_count(self, obj):
        return self._get_comments(obj).count() or 0

    def _get_comments(self, obj):
        return Comment.objects.filter(
            object_pk=obj.pk,
            content_type=get_post_content_type(),
            is_removed=False,
            site_id=settings.SITE_ID
        )

    def validate_tags(self, data):
        def _remove_spaces(str):
            return "".join(str.split())

        return [
            '#{}'.format(_remove_spaces(t))
            if not t.startswith('#')
            else _remove_spaces(t) for t in data]

    def _validate_image_link(self, value):
        url = value.lower()
        domain, path = split_url(url)

        # validate url first, image download will be done on the view
        if not valid_url_mimetype(url):
            _invalidate('Not a valid Image.')

    def _validate_video_link(self, value):
        pass

    def _validate_regular_link(self, value):
        pass

    def _validate_link(self, data):
        # set link validation depending on link type
        t = data['link_type']
        v = self._validate_image_link if t == Post.IMAGE \
            else self._validate_video_link if t == Post.VIDEO \
            else self._validate_regular_link

        v(data['link'])  # do the validation

    def validate(self, data):
        if 'image' not in data:  # if image wasnt uploaded
            self._validate_link(data)

        return data


import pandas as pd
# from .config import MAX_SEQ_LENGTH,FILE_NAME,REMOVE_NUMBERS
import re
from string import digits
import numpy as np
from sklearn.utils import shuffle
from sklearn.model_selection import train_test_split
from tensorflow.keras.preprocessing.text import one_hot, Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
import pickle as pkl

MAX_SEQ_LENGTH = 50 # very important to truncate or things will get too large and bog down the system

FILE_NAME = 'en-tw.txt' # Name of the file with data

REMOVE_NUMBERS = True

print('Data Processing Started')
lines= pd.read_table('data/'+FILE_NAME, names=['source', 'target'])

# check dimensions
# print('shape of data:'+lines.shape)

# Remove extra spaces
lines.source=lines.source.apply(lambda x: x.strip())
lines.target=lines.target.apply(lambda x: x.strip())
lines.source=lines.source.apply(lambda x: re.sub(" +", " ", x))
lines.target=lines.target.apply(lambda x: re.sub(" +", " ", x))

# Remove all numbers from text
if REMOVE_NUMBERS:

    remove_digits = str.maketrans('', '', digits)
    lines.source=lines.source.apply(lambda x: x.translate(remove_digits))
    lines.target=lines.target.apply(lambda x: x.translate(remove_digits))

# truncate to MAX_SEQ_LENGTH
lines.source=lines.source.apply(lambda x: " ".join(x.split(' ')[:MAX_SEQ_LENGTH]))
lines.target=lines.target.apply(lambda x: " ".join(x.split(' ')[:MAX_SEQ_LENGTH]))

# Add start and end tokens to target sequences
lines.target = lines.target.apply(lambda x : "start "+ x + " end")

# Vocabulary of Source
all_source_words=set()
for s in lines.source:
    for word in s.split():
        if word not in all_source_words:
            all_source_words.add(word)

# Vocabulary of Target
all_target_words=set()
for t in lines.target:
    for word in t.split():
        if word not in all_target_words:
            all_target_words.add(word)

# Max Length of source sequence
lenght_list=[]
for l in lines.source:
    lenght_list.append(len(l.split(' ')))
max_length_src = np.max(lenght_list)

# Max Length of target sequence
lenght_list=[]
for l in lines.target:
    lenght_list.append(len(l.split(' ')))
max_length_tar = np.max(lenght_list)

input_words = sorted(list(all_source_words))
target_words = sorted(list(all_target_words))
num_encoder_tokens = len(all_source_words)
num_decoder_tokens = len(all_target_words)

num_decoder_tokens += 1 # For zero padding
num_decoder_tokens

input_token_index = dict([(word, i+1) for i, word in enumerate(input_words)])
target_token_index = dict([(word, i+1) for i, word in enumerate(target_words)])

reverse_input_char_index = dict((i, word) for word, i in input_token_index.items())
reverse_target_char_index = dict((i, word) for word, i in target_token_index.items())

lines = shuffle(lines)

# Train - Test Split
X, y = lines.source, lines.target
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.1)

def Max_length(data):
    max_length_ = max([len(x.split(' ')) for x in data])
    return max_length_

#Training data
max_length_source = Max_length(X_train)
max_length_target = Max_length(y_train)

#Test data
max_length_source_test = Max_length(X_test)
max_length_target_test = Max_length(y_test)

sourceTokenizer = Tokenizer()
sourceTokenizer.fit_on_texts(X_train)
source_word2index = sourceTokenizer.word_index
vocab_size_source = len(source_word2index) + 1

X_train = sourceTokenizer.texts_to_sequences(X_train)
X_train = pad_sequences(X_train, maxlen=max_length_source, padding='post')

X_test = sourceTokenizer.texts_to_sequences(X_test)
X_test = pad_sequences(X_test, maxlen = max_length_source, padding='post')

targetTokenizer = Tokenizer()
targetTokenizer.fit_on_texts(y_train)
targetword2index = targetTokenizer.word_index
vocab_size_target = len(targetword2index) + 1

y_train = targetTokenizer.texts_to_sequences(y_train)
y_train = pad_sequences(y_train, maxlen=max_length_target, padding='post')

y_test = targetTokenizer.texts_to_sequences(y_test)
y_test = pad_sequences(y_test, maxlen = max_length_target, padding='post')

with open('processed_data/nmt_data.pkl','wb') as f:
  pkl.dump([X_train, y_train, X_test, y_test,max_length_source,max_length_target],f)

with open('processed_data/nmt_source_tokenizer.pkl','wb') as f:
  pkl.dump([vocab_size_source, source_word2index, sourceTokenizer], f)

with open('processed_data/nmt_target_tokenizer.pkl', 'wb') as f:
  pkl.dump([vocab_size_target, targetword2index, targetTokenizer], f)

print('Data Processing Finished')


import subprocess
from pulumi import automation as auto
from app_util.config import TestConfig
from util.logging import PulumiLogging, ServiceLogging, grouped_logging
import logging
import os
import boto3

log = logging.getLogger("DeploymentRunner")

class AppDeployer:
    def __init__(self, region: str, pulumi_logger: PulumiLogging):
        self.region = region
        self.pulumi_logger = pulumi_logger
        self.logs_client= boto3.client('logs', region_name=self.region)

    def set_stack(self, stack: auto.Stack):
        self.stack = stack

    def configure_and_deploy(self, cfg: TestConfig) -> str:
        with grouped_logging("Configuring stack"):
            self.configure_tags()
            self.configure_region()
            self.configure_pulumi_app(cfg)
        for i in range(0,5):
            with grouped_logging(f'Previewing stack {self.stack.name}, attempt #{i}'):
                try:
                    self.stack.preview(on_output=self.pulumi_logger.log)
                    break
                except Exception as e:
                    log.error(f'Failed to preview stack', exc_info=True)
                    if i == 4:
                        return ""
        for i in range(0, 5):
            try:
                with grouped_logging(f'Deploying stack {self.stack.name} to region {self.region}, attempt #{i}'):
                    url = self.deploy_app()
                    log.info(f'Deployed stack, {self.stack.name}, successfully. Got API Url: {url}')
                    return url
            except Exception as e:
                with grouped_logging(f'Deployment of stack, {self.stack.name}, failed.'):
                    log.error(f'Deployment of stack, {self.stack.name}, failed.', exc_info=True)
                    log.info(f'Refreshing stack {self.stack.name}')
                    self.stack.refresh()
        return ""

    # deploy_app deploys the stack and returns the first apiUrl expecting it to be the intended endpoint for tests
    def deploy_app(self) -> str:
        result: auto.UpResult = self.stack.up(on_output=self.pulumi_logger.log)
        output: auto.OutputMap = result.outputs
        value: auto.OutputValue = output.get("apiUrls")
        if len(value.value) > 0:
            if type(value.value[0]) is str:
                return value.value[0]
        return ""

    def configure_tags(self):
        result: subprocess.CompletedProcess[bytes] = subprocess.run(
            ["pulumi", "stack", "tag", "set", "usage", "infra-test", "-s", self.stack.name],
            stdout=open(os.devnull, 'wb'),
            cwd=self.stack.workspace.work_dir
        )
        result.check_returncode()

    def configure_region(self):
        self.stack.set_config("aws:region", auto.ConfigValue(self.region))

    def configure_pulumi_app(self, cfg: TestConfig):
        for key in cfg.pulumi_config:
            self.stack.set_config(f'{cfg.app_name}:{key}', auto.ConfigValue(cfg.pulumi_config[key]))

    def destroy_and_remove_stack(self, output_dir: str) -> bool:
         for i in range(0,5):
            try:
                with grouped_logging(f'Destroying stack {self.stack.name}, attempt #{i}'):
                    self.pulumi_logger.set_file_name(f'destroy.txt')
                    self.stack.destroy(on_output=self.pulumi_logger.log)
                    log.info(f'Removing stack {self.stack.name}')
                    command = f'cd {output_dir}; pulumi stack rm -s {self.stack.name} -y'
                    result: subprocess.CompletedProcess[bytes] = subprocess.run(command, capture_output=True, shell=True)
                    result.check_returncode()
                    return True
            except Exception as e:
                with grouped_logging("Destroy failed"):
                    log.error(e)
                    log.info(f'Refreshing stack {self.stack.name}')
                    self.stack.refresh()
                    return False

    def download_logs(self, logger: ServiceLogging):
        deployment = self.stack.export_stack()
        resources = deployment.deployment.get('resources')
        for resource in resources:

            # This covers lambda and ecs since we explicitly create the log groups for both
            if resource['type'] == 'aws:cloudwatch/logGroup:LogGroup':
                log_group_name: str = resource['outputs']['name']
                log.info(f'Getting logs from {log_group_name}')
                log_streams = self.logs_client.describe_log_streams(
                    logGroupName=log_group_name,
                    limit=50
                )
                for stream in log_streams['logStreams']:
                    stream_name: str = stream['logStreamName']
                    events = self.get_log_events(log_group_name, stream_name)
                    logger.write_file(f'{log_group_name.replace("/", "_")}-{stream_name.replace("/", "_")}', events)

            elif resource['type'] == 'aws:eks/cluster:Cluster':
                cluster_name = resource['outputs']['name']
                log_group_name: str = f'/aws/containerinsights/{cluster_name}/application'
                log.info(f'Getting logs from {log_group_name}')
                log_streams = self.logs_client.describe_log_streams(
                    logGroupName=log_group_name,
                    limit=50
                )
                for stream in log_streams['logStreams']:
                    stream_name: str = stream['logStreamName']
                    events = self.get_log_events(log_group_name, stream_name)
                    logger.write_file(stream_name.replace("/", "_"), events)

    def get_log_events(self, log_group_name: str, stream: str):
        kwargs = {
            'logGroupName': log_group_name,
            'limit': 50,
            'logStreamName': stream
        }
        events = []
        while True:
            resp = self.logs_client.get_log_events(**kwargs)
            try:
                [events.append(e['message']) for e in resp['events']]
                kwargs['nextToken'] = resp['nextToken']
            except KeyError:
                break
        return events


import numpy as np

import theano
floatX = theano.config.floatX

class AgentEpsGreedy:

    def __init__(self, n_actions, value_function_model, state_dim, batch_size, eps=0.1):
        self.n_actions = n_actions
        self.value_func = value_function_model
        self.eps = eps
        self.state_dim = state_dim
        self.batch_size = batch_size

    def act(self, state):

        state = np.array([state])
        action_values = self.value_func.predict(state)

        policy = np.ones(self.n_actions) * self.eps / self.n_actions
        a_max = np.argmax(action_values)
        policy[a_max] += 1. - self.eps

        return np.random.choice(self.n_actions, p=policy)

    def thompson_hypernet_act(self, state):

        state = np.array([state])
        dropout_iterations=100
        dropout_acton_values = np.zeros(shape=(self.n_actions, dropout_iterations))

        for d in range(dropout_iterations):
            action_values = self.value_func.predict(state.astype(floatX))
            dropout_acton_values[:, d] = action_values

        mean_action_values = np.mean(dropout_acton_values, axis=1)

        return np.argmax(mean_action_values)

    def act_hypernet_EpsilonGreedy(self, state):

        state = np.array([state])

        if np.random.rand() < self.eps:
            return np.random.randint(self.n_actions)

        else:
            dropout_iterations=100
            dropout_acton_values = np.zeros(shape=(self.n_actions, dropout_iterations))

            for d in range(dropout_iterations):
                action_values = self.value_func.predict(state.astype(floatX))
                dropout_acton_values[:, d] = action_values

            mean_action_values = np.mean(dropout_acton_values, axis=1)

            return np.argmax(mean_action_values)

    def act_hypernet_epsilon_entropy(self, state):
        state = np.array([state])
        dropout_iterations=100

        if np.random.rand() < self.eps:

            dropout_acton_values_entropy = np.zeros(shape=(self.n_actions, dropout_iterations))

            for d in range(dropout_iterations):
                action_values_entropy = self.value_func.predict(state.astype(floatX))
                dropout_acton_values_entropy[:, d] = action_values_entropy

            mean_action_values_entropy = np.mean(dropout_acton_values_entropy, axis=1)
            log_mean =  np.log2(mean_action_values_entropy)

            Entropy_Average_Pi = - np.multiply(mean_action_values_entropy, log_mean)
            max_entropy_action = np.argmax(Entropy_Average_Pi)

            return max_entropy_action

        else:

            dropout_acton_values = np.zeros(shape=(self.n_actions, dropout_iterations))

            for d in range(dropout_iterations):
                action_values = self.value_func.predict(state.astype(floatX))
                dropout_acton_values[:, d] = action_values

            mean_action_values = np.mean(dropout_acton_values, axis=1)

            return np.argmax(mean_action_values)

    # TODO: more float problems here???
    def eval_train(self, states, targets, dropout_probability):
        return self.value_func.eval_train(states.astype(floatX), targets.astype(floatX), dropout_probability)

    def eval_valid(self, states, targets, dropout_probability):
        return self.value_func.eval_valid(states.astype(floatX), targets.astype(floatX), dropout_probability)

    def predict_q_values(self, states):
        return self.value_func.predict(states.astype(floatX))

    def evaluate_predicted_q_values(self, states, dropout_probability):
        return self.value_func.predict_stochastic(states.astype(floatX), dropout_probability)

    #good lr0 : 0.0001
    def train(self, X,Y, lr0=0.001,lrdecay=1,bs=20,epochs=50):
        X = X.astype(floatX)
        Y = Y.astype(floatX)

        train_func = self.value_func.train_func

        N = X.shape[0]

        for e in range(epochs):

            if lrdecay:
                lr = lr0 * 10**(-e/float(epochs-1))
            else:
                lr = lr0

            for i in range(N/bs):
                x = X[i*bs:(i+1)*bs]
                y = Y[i*bs:(i+1)*bs]

                loss = train_func(x,y,N,lr)

        return loss

    def act_boltzmann(self, state):
        action_values = self.value_func.predict([state])[0]
        action_values_tau = action_values / self.eps
        policy = np.exp(action_values_tau) / np.sum(np.exp(action_values_tau), axis=0)
        action_value_to_take = np.argmax(policy)
        return action_value_to_take


import os

from dotenv import load_dotenv

class Config:
    """Class for environment configurations. Load from .env file
    with environment specific variables prefixed with environment name.

    Example: AWS_ROLE_CLOUD_ADMIN=<REDACTED>
    """

    def __init__(self) -> None:
        load_dotenv()
        self.aws_role_cloud_admin: str = os.environ.get("AWS_ROLE_CLOUD_ADMIN")
        self.aws_role_adfs_admin: str = os.environ.get("AWS_ROLE_ADFS_ADMIN")

    def __str__(self) -> str:
        """Beautify when printing the class

        :return: str
        """
        output = (
            f"AWS Cloud Admin Role ARN: {self.aws_role_cloud_admin}\n"
            f"AWS ADFS Admin Role ARN: {self.aws_role_adfs_admin}"
        )
        return output


import claripy
import logging

from . import MemoryMixin
from ... import sim_options as o

l = logging.getLogger(__name__)

class UnderconstrainedMixin(MemoryMixin):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

        self._unconstrained_range = 1024

    @MemoryMixin.memo
    def copy(self, memo):
        out = super().copy(memo)
        out._unconstrained_range = self._unconstrained_range
        return out

    def load(self, addr, **kwargs):
        self._constrain_underconstrained_index(addr)
        return super().load(addr, **kwargs)

    def store(self, addr, data, **kwargs):
        self._constrain_underconstrained_index(addr)
        super().store(addr, data, **kwargs)

    def _default_value(self, addr, size, name=None, key=None, inspect=True, events=True, **kwargs):
        if o.UNDER_CONSTRAINED_SYMEXEC in self.state.options and type(addr) is int:
            if self.category == "mem":
                alloc_depth = self.state.uc_manager.get_alloc_depth(addr)
                uc_alloc_depth = 0 if alloc_depth is None else alloc_depth + 1
            else:
                uc_alloc_depth = 0

            if name is None:
                name = "mem_%x" % addr

            bits = size * self.state.arch.byte_width
            return self.state.solver.Unconstrained(
                name, bits, key=key, inspect=inspect, events=events, uc_alloc_depth=uc_alloc_depth
            )

        return super()._default_value(addr, size, name=name, key=key, inspect=inspect, events=events, **kwargs)

    def _constrain_underconstrained_index(self, addr):
        if (
            o.UNDER_CONSTRAINED_SYMEXEC in self.state.options
            and isinstance(addr, claripy.ast.Base)
            and addr.uninitialized
            and addr.uc_alloc_depth is not None
        ):
            if (
                not self.state.uc_manager.is_bounded(addr)
                or self.state.solver.max_int(addr) - self.state.solver.min_int(addr) >= self._unconstrained_range
            ):
                # in under-constrained symbolic execution, we'll assign a new memory region for this address
                mem_region = self.state.uc_manager.assign(addr)

                # ... but only if it's not already been constrained to something!
                if self.state.solver.solution(addr, mem_region):
                    self.state.add_constraints(addr == mem_region)
                l.debug(
                    "Under-constrained symbolic execution: assigned a new memory region @ %s to %s", mem_region, addr
                )


from collections import defaultdict, namedtuple
from contrib.performance.stats import LogNormalDistribution
from scipy.optimize import curve_fit
import itertools
import matplotlib.pyplot as plt
import numpy as np
import os

class LogNormal(object):

    Params = namedtuple("Params", ("mu", "sigma", "scale_x", "scale_y"))

    @staticmethod
    def fn(x, mu, sigma, scale_x, scale_y):
        """
        Calculate the LogNormal F(x) value for the given parameters.

        @param x: x value to use
        @type x: L{float}
        @param mu: mean
        @type mu: L{float}
        @param sigma: standard deviation
        @type sigma: L{float}
        @param scale_x: X-scaling factor (to make mode == 1.0)
        @type scale_x: L{float}
        @param scale_y: Y-scaling factor for peak height
        @type scale_y: L{float}
        """
        return scale_y * (
            np.exp(-(np.log(x / scale_x) - mu) ** 2 / (2 * sigma ** 2)) /
            (x / scale_x * sigma * np.sqrt(2 * np.pi))
        )

    @staticmethod
    def estimate(x, y):
        """
        Given a set of x-y data that is likely a LogNormal distribution, try and estimate the mode
        and median, and then derive mu, sigma, scale_x and scale_y values.

        @param x: sequence of values
        @type x: L{list} of L{float}
        @param y: sequence of values
        @type y: L{list} of L{float}
        """
        estimate_mode = 0.0
        estimate_median = 0.0
        max_y = 0.0
        accumulated_y = 0.0
        half_y = sum(y) / 2.0
        for x_val, y_val in itertools.izip(x, y):
            if y_val > max_y:
                max_y = y_val
                estimate_mode = x_val
            accumulated_y += y_val
            if half_y is not None and accumulated_y > half_y:
                estimate_median = x_val
                half_y = None

        estimate_scale_x = estimate_mode - 0.5
        estimate_mode = 1.0
        estimate_median /= estimate_scale_x
        estimate_mu = np.log(estimate_median)
        estimate_sigma = np.sqrt(np.log(estimate_median) - np.log(estimate_mode))

        peak_y = LogNormal.fn(estimate_scale_x, estimate_mu, estimate_sigma, estimate_scale_x, 1.0)
        estimate_scale_y = max(y) / peak_y
        return (
            estimate_mode, estimate_median,
            LogNormal.Params(estimate_mu, estimate_sigma, estimate_scale_x, estimate_scale_y),
        )

    @staticmethod
    def plot(min_x, max_x, params, color):
        """
        Plot a LogNormal distribution over the specified x-axis range using the supplied
        distribution parameters.

        @param min_x: minimum x-axis value
        @type min_x: L{float}
        @param max_x: maximum x-asix value
        @type max_x: L{float}
        @param params: distribution parameters
        @type params: L{LogNormal.Params}
        @param color: color to use for line in plot
        @type color: L{str}
        """
        xl = np.linspace(min_x, max_x, 10000)
        yl = (
            np.exp(-(np.log(xl / params.scale_x) - params.mu) ** 2 / (2 * params.sigma ** 2)) /
            (xl / params.scale_x * params.sigma * np.sqrt(2 * np.pi))
        )

        plt.plot(xl, params.scale_y * yl, linewidth=2, color=color)

    @staticmethod
    def plotCSV(path, cutoff, bucket, color):
        """
        Plot data from a CSV file.

        @param path: file to read from
        @type path: L{str}
        @param cutoff: maximum x-value to process
        @type cutoff: L{float}
        @param bucket: size of x-value buckets to use
        @type bucket: L{int}
        @param color: color to use for line in plot
        @type color: L{str}
        """
        with open(os.path.expanduser(path)) as f:
            data = f.read()
        result = defaultdict(int)
        for line in data.splitlines():
            sp = line.split(",")
            x_val = int(sp[0])
            y_val = int(sp[1])
            if x_val < cutoff:
                result[(x_val / bucket) * bucket] += y_val

        x, y = zip(*sorted(result.items()))
        plt.plot(x, y)
        return (x, y,)

    @staticmethod
    def distributionPlot(mode, median, maximum, color):
        """
        Plot data from a randomly generated LogNornal distribution.

        @param mode: distribution mode
        @type mode: L{float}
        @param median: distribution median
        @type median: L{float}
        @param maximum: highest x-value to allow
        @type maximum: L{float}
        @param color: color to use for line in plot
        @type color: L{str}
        """
        distribution = LogNormalDistribution(mode=mode, median=median, maximum=maximum)
        result = defaultdict(int)
        for _ignore in range(1000000):
            s = int(distribution.sample()) + 0.5
            result[s] += 1

        x, y = zip(*sorted(result.items()))
        plt.plot(x, y, color=color)

        peak_y = LogNormal.fn(distribution._scale, distribution._mu, distribution._sigma, distribution._scale, 1.0)
        scale_y = sum(sorted(y, reverse=True)[0:100]) / 100.0 / peak_y

        return (x, y, LogNormal.Params(distribution._mu, distribution._sigma, distribution._scale, scale_y),)

    @staticmethod
    def fit(x, y):
        """
        Try and fit a LogNormal distribution to the supplied x-y data.

        @param x: sequence of values
        @type x: L{list} of L{float}
        @param y: sequence of values
        @type y: L{list} of L{float}
        """
        estimate_mode, estimate_median, estimate_params = LogNormal.estimate(x, y)

        print("\n==== Estimates")
        print("mode: {}".format(estimate_mode * estimate_params.scale_x))
        print("median: {}".format(estimate_median * estimate_params.scale_x))
        print("mu: {}".format(estimate_params.mu))
        print("sigma: {}".format(estimate_params.sigma))
        print("scale_x: {}".format(estimate_params.scale_x))
        print("scale_y: {}".format(estimate_params.scale_y))

        popt, _ignore_pcov = curve_fit(LogNormal.fn, x, y, (
            estimate_params.mu, estimate_params.sigma, estimate_params.scale_x, estimate_params.scale_y,
        ))

        print("\n==== Fit results")
        print("mode: {:.2f}".format(popt[2] * np.exp(popt[0] - popt[1] ** 2)))
        print("median: {:.2f}".format(popt[2] * np.exp(popt[0])))
        print("mu: {:.2f}".format(popt[0]))
        print("sigma: {:.2f}".format(popt[1]))
        print("scale_x: {:.2f}".format(popt[2]))
        print("scale_y: {:.2f}".format(popt[3]))

        return (
            LogNormal.Params(*popt),
            estimate_params,
        )

if __name__ == '__main__':

    #x, y = LogNormal.plotCSV("~/data.txt", 10000, 10, color="b")
    #estimate_mode, estimate_median, estimate_params = LogNormal.estimate(x, y)

    mode_val = 450
    median_val = 650
    x, y, estimate_params = LogNormal.distributionPlot(mode_val, median_val, 10000, color="b")

    LogNormal.plot(1, 10000, estimate_params, color="g")

    popt, estimate = LogNormal.fit(x, y)
    LogNormal.plot(1, 10000, popt, color="r")

    plt.xlabel("Samples")
    plt.ylabel("LogNormal")
    plt.show()


import pandas as pd
import numpy as np
import scipy as sp
import matplotlib.pyplot as plt
from scipy import stats
from pyextremes import plot_mean_residual_life
import math
from scipy.stats import kstest
# =============================================================================
#Importing files
# =============================================================================
df=pd.read_excel('h:\Downloads\Données.xlsx')
data=df.loc[:,'Y']
data1= df[["date", "Y"]]
data1= data1.set_index("date")
data1=data1.resample("Y").max()
# =============================================================================
# Pareto threshold selection and histograms
# =============================================================================

plot_mean_residual_life(data)
plt.title("Mean Excess Plot using Pyextremes Library")
plt.show()
datapar=[x for x in df.loc[:,'Y']  if x>=3800]
v=np.ceil(np.log2(len(data))) + 1
v2=np.ceil(np.log2(len(data1))) + 1
v3=np.ceil(np.log2(len(datapar))) + 1 #This is called Sturge's rule, this formula is used to calculate the adequate number of bins to visualize your data's distribution
y,x=np.histogram(data,bins=int(v),density=True)
y2,x2=np.histogram(data1,bins=int(v2),density=True)
y3,x3=np.histogram(datapar,bins=int(v3),density=True) #"clustering" our data for the plot
x = (x + np.roll(x, -1))[:-1] / 2.0
x2 = (x2 + np.roll(x2, -1))[:-1] / 2.0
x3 = (x3 + np.roll(x3, -1))[:-1] / 2.0#This takes the mid point of every "bin" interval as the reference x-axis point for its corresponding y probability
# =============================================================================
# Fitting our data and plotting the PDFs
# =============================================================================
fit1=stats.genextreme.fit(data1,loc=8) #The fit method finds the optimal parameters for your data fitting a chosen probability distribution
fit2=stats.norm.fit(data)
fit3=stats.genpareto.fit(datapar,loc=8)
pdf=stats.genextreme.pdf(x2,*fit1)
pdf2=stats.norm.pdf(x,*fit2)
pdf3=stats.genpareto.pdf(x3,*fit3)
plt.plot(x,pdf2,c='r',label='Gaussian Law')
plt.plot(x,y,c='g',label='Real Data ')
plt.legend()
plt.title(" Data and Gaussian densities")
plt.xlim(3400,5000)
plt.ylim(0,0.001)
plt.show()
plt.plot(x2,pdf,c='b',label='GEV')
plt.plot(x2,y2,c='g',label='Real Data ')
plt.legend()
plt.title("Data and GEV probability densities")
plt.show()
plt.plot(x3,pdf3,c='y',label='GPD')
plt.plot(x3,y3,c='g',label='Real Data ')
plt.legend()
plt.title(" Data and GPD probability densities")
plt.show()

# =============================================================================
# Finding the best fit using least-squares
# =============================================================================
def empirical(x,u):
    count=0
    for i in range(len(x)):
        if x[i]>u:
            count+=1

    return count/len(x)
#In order to compare with our Gaussian/GEV/GPD distributions we will simulate values from those and compare with our empirical distribution
Xaxis=np.linspace(3000,6000,50)
YaxisEmpiricalDistribution=np.zeros(len(Xaxis))
YaxisGaussian=np.zeros(len(Xaxis))
YaxisGev=np.zeros(len(Xaxis))
YaxisGPD=np.zeros(len(Xaxis))
dataGaussian=stats.norm.rvs(*fit2,size=len(data)) #we generate len(data) number of values from our distributions and plot their respective probabilities
dataGPD=stats.genpareto.rvs(*fit3,size=1000)
dataGev=stats.genextreme.rvs(*fit1,size=1000)
YaxisEmpiricalDistributionYear=np.zeros(len(Xaxis))
for i in range(len(Xaxis)):
    YaxisEmpiricalDistribution[i]=empirical(data,Xaxis[i])
    YaxisEmpiricalDistributionYear[i]=empirical(np.array(data1),Xaxis[i])
    YaxisEmpiricalDistributionYear[i]=empirical(datapar,Xaxis[i])
    YaxisGaussian[i]=empirical(dataGaussian,Xaxis[i])
    YaxisGev[i]=empirical(dataGev,Xaxis[i])
    YaxisGPD[i]=empirical(dataGPD,Xaxis[i])

plt.plot(Xaxis,YaxisEmpiricalDistribution,c='g',label='Original Data Disitribution')
plt.legend()
plt.title("Empirical distribution P(X>u)=1-P(X<u) ")
plt.show()
plt.plot(Xaxis,YaxisEmpiricalDistribution,c='g',label='Original Data Sample')
plt.plot(Xaxis,YaxisGaussian,c='b',label='Scaled Gaussian Sample')
plt.legend()
plt.title("Empirical distribution P(X>u)=1-P(X<u) Gaussian and Original Data ")
plt.show()
plt.plot(Xaxis,YaxisEmpiricalDistributionYear,c='g',label='Original Data Sample')
plt.plot(Xaxis,YaxisGPD,c='r',label='GPD Data Sample')
plt.title("Empirical distribution P(X>u)=1-P(X<u) GPD and Original Data")
plt.legend()
plt.show()
plt.plot(Xaxis,YaxisEmpiricalDistributionYear,c='g',label='Original Data Sample')
plt.plot(Xaxis,YaxisGev,c='y',label='GEV Data Sample')
plt.title("Empirical distribution P(X>u)=1-P(X<u) GEV and Original Data")
plt.legend()
plt.show()
plt.plot(Xaxis,YaxisEmpiricalDistribution,c='g',label='Original Data Sample')
plt.plot(Xaxis,YaxisGaussian,c='b',label='Gaussian')
plt.plot(Xaxis,YaxisGPD,c='r',label='GPD')
plt.plot(Xaxis,YaxisGev,c='y',label='GEV')
plt.title("All distributions plotted together")
plt.legend()
plt.show()
#Least-squares
def leastsquares(Dist): #Function to find the best fitting model
    Error=0
    for i in range(len(Xaxis)):
        Error+=(YaxisEmpiricalDistribution[i]-Dist[i])**2
    return Error/len(Xaxis)
#Kolmogorov-Smirnoff
x=kstest(data,stats.norm.cdf,args=fit2)
x2=kstest(datapar,stats.genpareto.cdf,args=fit3)
print("Kstest for gaussian",x) #P value is pretty high we cannot reject the gaussian distribution, also from the p value from the GPD, the tail seems to be more gaussian than anything else
print("Kstest for GPD",x2)
def Quantile(alpha,data):
    Sortedata=np.sort(data)
    IndexData=math.floor(len(data)*alpha)
    return Sortedata[IndexData]
ReturnPeriods=[12*i for i in range(2,11)]
Alphas=[1-1/x for x in ReturnPeriods]
Alphas2=[1-12/x for x in ReturnPeriods]
Alphas3=[1-12/x for x in ReturnPeriods]

Quantiledata=np.zeros(len(Alphas))
QuantileGEV=np.zeros(len(Alphas))
QuantileGPD =np.zeros(len(Alphas))
QuantileGauss=np.zeros(len(Alphas))

for i in range(len(Alphas)):
    Quantiledata[i]=Quantile(Alphas[i],data)
    QuantileGauss[i]=Quantile(Alphas[i],stats.norm.rvs(*fit2,100000))
    QuantileGEV[i]=Quantile(Alphas2[i],stats.genextreme.rvs(*fit1,100000))
    QuantileGPD[i]=Quantile(Alphas3[i],stats.genpareto.rvs(*fit3,100000))

plt.scatter(Quantiledata,QuantileGauss)
plt.plot(Quantiledata,Quantiledata)
plt.title("Q-Q plot Gauss and Original Data")
plt.show()
plt.scatter(Quantiledata,QuantileGEV)
plt.plot(Quantiledata,Quantiledata)
plt.title("Q-Q plot GEV Original Data")
plt.show()
plt.scatter(Quantiledata,QuantileGPD)
plt.plot(Quantiledata,Quantiledata)
plt.title("Q-Q plot GPD Original Data")
plt.show()
#We select the Gaussian model
# =============================================================================
#  Quantiles
# =============================================================================
#Actual Data
SortedData=np.sort(df.loc[:,'Y'])
alpha=0.995
alphagauss=1-1/(12*200)
alpha2=1-(12/200)
IndexData=math.floor(len(df.loc[:,'Y'])*alpha)
QuantileData=SortedData[IndexData]
#Gaussian law
SortedGaussian=np.sort(stats.norm.rvs(*fit2,size=1000000))
IndexGauss=math.floor(1000000*alphagauss)
IndexQuantileGauss=math.floor(1000000*alpha)
QuantileGaussian=SortedGaussian[IndexGauss]
QuantileGaussian99=SortedGaussian[IndexQuantileGauss]
print("Modèle Y Quantile 99.5 de la loi normale  est", QuantileGaussian99)
#GPD
SortedGPD=np.sort(stats.genpareto.rvs(*fit3,size=100000))
IndexGPD=math.floor(100000*alpha)
IndexGPDQuantile=math.floor(100000*alpha2)
ReturnPareto=SortedGPD[IndexGPD]
QuantilePareto=SortedGPD[IndexGPDQuantile]
#GEV
SortedGEV=np.sort(stats.genextreme.rvs(*fit1,size=100000))
IndexGEV=math.floor(100000*alpha)
IndexGEVQuantile=math.floor(100000*alpha2)
ReturnGEV=SortedGEV[IndexGEV]
QuantileGEV=SortedGEV[IndexGEVQuantile]
QuantileVector=[QuantileData,QuantileGEV,QuantilePareto]
print("Model Y Quantile 99.5 OriginalData,GEV, and GPD are", QuantileVector)
ReturnVector=[QuantileGaussian,ReturnGEV,ReturnPareto]
print("Model Y 200 years return period  for Gaussian,GEV, and GPD are", ReturnVector)


import torch
import torch.nn as nn
import torch.nn.functional as F

from utils.network_utils import *

class ImageDown(torch.nn.Module):
    def __init__(self, input_size, num_chan, out_dim):
        super(ImageDown, self).__init__()
        self.__dict__.update(locals())

        norm_layer = functools.partial(nn.BatchNorm2d, affine=True)
        activ = nn.LeakyReLU(0.2, True)

        _layers = []
        if input_size == 64:
            cur_dim = 128
            _layers += [conv_norm(num_chan, cur_dim, norm_layer, stride=2, activation=activ, use_norm=False)]  # 32
            _layers += [conv_norm(cur_dim, cur_dim*2, norm_layer, stride=2, activation=activ)]  # 16
            _layers += [conv_norm(cur_dim*2, cur_dim*4, norm_layer, stride=2, activation=activ)]  # 8
            _layers += [conv_norm(cur_dim*4, out_dim,  norm_layer, stride=1, activation=activ, kernel_size=5, padding=0)]  # 4

        if input_size == 128:
            cur_dim = 64
            _layers += [conv_norm(num_chan, cur_dim, norm_layer, stride=2, activation=activ, use_norm=False)]  # 64
            _layers += [conv_norm(cur_dim, cur_dim*2, norm_layer, stride=2, activation=activ)]  # 32
            _layers += [conv_norm(cur_dim*2, cur_dim*4, norm_layer, stride=2, activation=activ)]  # 16
            _layers += [conv_norm(cur_dim*4, cur_dim*8, norm_layer, stride=2, activation=activ)]  # 8
            _layers += [conv_norm(cur_dim*8, out_dim,  norm_layer, stride=1, activation=activ, kernel_size=5, padding=0)]  # 4

        if input_size == 256:
            cur_dim = 32 # for testing
            _layers += [conv_norm(num_chan, cur_dim, norm_layer, stride=2, activation=activ, use_norm=False)] # 128
            _layers += [conv_norm(cur_dim, cur_dim*2,  norm_layer, stride=2, activation=activ)] # 64
            _layers += [conv_norm(cur_dim*2, cur_dim*4,  norm_layer, stride=2, activation=activ)] # 32
            _layers += [conv_norm(cur_dim*4, cur_dim*8,  norm_layer, stride=2, activation=activ)] # 16
            _layers += [conv_norm(cur_dim*8, out_dim,  norm_layer, stride=2, activation=activ)] # 8

        self.node = nn.Sequential(*_layers)

    def forward(self, inputs):
        out = self.node(inputs)
        return out

class DiscClassifier(nn.Module):
    def __init__(self, enc_dim, emb_dim, kernel_size):
        super(DiscClassifier, self).__init__()
        self.__dict__.update(locals())
        norm_layer = functools.partial(nn.BatchNorm2d, affine=True)
        activ = nn.LeakyReLU(0.2, True)
        inp_dim = enc_dim + emb_dim

        _layers = [conv_norm(inp_dim, enc_dim, norm_layer, kernel_size=1, stride=1, activation=activ),
                   nn.Conv2d(enc_dim, 1, kernel_size=kernel_size, padding=0, bias=True)]

        self.node = nn.Sequential(*_layers)

    def forward(self, sent_code,  img_code):
        sent_code = sent_code.unsqueeze(-1).unsqueeze(-1)
        dst_shape = list(sent_code.size())
        dst_shape[1] = sent_code.size()[1]
        dst_shape[2] = img_code.size()[2]
        dst_shape[3] = img_code.size()[3]
        sent_code = sent_code.expand(dst_shape)
        comp_inp = torch.cat([img_code, sent_code], dim=1)
        output = self.node(comp_inp)
        chn = output.size()[1]
        output = output.view(-1, chn)

        return output

class DISCRIMINATOR(torch.nn.Module):
    def __init__(self, num_chan,  hid_dim, sent_dim, emb_dim, side_output_at=[64, 128, 256]):
        super(DISCRIMINATOR, self).__init__()
        self.__dict__.update(locals())

        activ = nn.LeakyReLU(0.2, True)
        norm_layer = functools.partial(nn.BatchNorm2d, affine=True)

        self.side_output_at = side_output_at

        enc_dim = hid_dim * 4  # the ImageDown output dimension

        if 64 in side_output_at:  # discriminator for 64 input
            self.img_encoder_64 = ImageDown(64,  num_chan,  enc_dim)  # 4x4
            self.pair_disc_64 = DiscClassifier(enc_dim, emb_dim, kernel_size=4)
            self.local_img_disc_64 = nn.Conv2d(enc_dim, 1, kernel_size=4, padding=0, bias=True)
            _layers = [nn.Linear(sent_dim, emb_dim), activ]
            self.context_emb_pipe_64 = nn.Sequential(*_layers)

        if 128 in side_output_at:  # discriminator for 128 input
            self.img_encoder_128 = ImageDown(128,  num_chan, enc_dim)  # 4x4
            self.pair_disc_128 = DiscClassifier(enc_dim, emb_dim, kernel_size=4)
            self.local_img_disc_128 = nn.Conv2d(enc_dim, 1, kernel_size=4, padding=0, bias=True)
            _layers = [nn.Linear(sent_dim, emb_dim), activ]
            self.context_emb_pipe_128 = nn.Sequential(*_layers)

        if 256 in side_output_at:  # discriminator for 256 input
            self.img_encoder_256 = ImageDown(256, num_chan, enc_dim)     # 8x8
            self.pair_disc_256 = DiscClassifier(enc_dim, emb_dim, kernel_size=4)
            self.pre_encode = conv_norm(enc_dim, enc_dim, norm_layer, stride=1, activation=activ, kernel_size=5, padding=0)
            self.local_img_disc_256 = nn.Conv2d(enc_dim, 1, kernel_size=4, padding=0, bias=True)
            _layers = [nn.Linear(sent_dim, emb_dim), activ]
            self.context_emb_pipe_256 = nn.Sequential(*_layers)

        self.apply(weights_init)

    def forward(self, images, embedding):
        out_dict = OrderedDict()
        this_img_size = images.size()[3]
        assert this_img_size in [32, 64, 128, 256], 'wrong input size {} in image discriminator'.format(this_img_size)

        img_encoder = getattr(self, 'img_encoder_{}'.format(this_img_size))
        local_img_disc = getattr(
            self, 'local_img_disc_{}'.format(this_img_size), None)
        pair_disc = getattr(self, 'pair_disc_{}'.format(this_img_size))
        context_emb_pipe = getattr(
            self, 'context_emb_pipe_{}'.format(this_img_size))

        sent_code = context_emb_pipe(embedding)
        img_code = img_encoder(images)

        if this_img_size == 256:
            pre_img_code = self.pre_encode(img_code)
            pair_disc_out = pair_disc(sent_code, pre_img_code)
        else:
            pair_disc_out = pair_disc(sent_code, img_code)

        local_img_disc_out = local_img_disc(img_code)

        return pair_disc_out, local_img_disc_out