from glob import glob
from collections import defaultdict
from string import punctuation
from functools import lru_cache

import kenlm
import nltk
from qanta.pattern3 import pluralize
from nltk.tokenize import word_tokenize

from qanta.extractors.abstract import AbstractFeatureExtractor
from qanta.util.environment import data_path, QB_QUESTION_DB
from qanta.util.constants import KEN_LM, MIN_APPEARANCES
from qanta.util.build_whoosh import text_iterator
from qanta.datasets.quiz_bowl import QuestionDatabase
from qanta.wikipedia.cached_wikipedia import CachedWikipedia
from clm.lm_wrapper import kTOKENIZER, LanguageModelBase

from nltk.corpus import wordnet as wn

@lru_cache(maxsize=None)
def get_states():
    states = set()
    for ii in wn.synset("American_state.n.1").instance_hyponyms():
        for jj in ii.lemmas():
            name = jj.name()
            if len(name) > 2 and "_" not in name:
                states.add(name)
            elif name.startswith("New_"):
                states.add(name.replace("New_", ""))
    return states

def find_references(sentence, padding=5):
    tags = nltk.pos_tag(word_tokenize(sentence))
    tags.append(("END", "V"))
    states = get_states()

    references_found = []
    this_ref_start = -1
    for ii, pair in enumerate(tags):
        word, tag = pair
        if word.lower() == 'this' or word.lower() == 'these':
            this_ref_start = ii
        elif all(x in punctuation for x in word):
            continue
        elif word in states:
            continue
        elif this_ref_start >= 0 and tag.startswith('NN') and \
                not tags[ii + 1][1].startswith('NN'):
            references_found.append((this_ref_start, ii))
            this_ref_start = -1
        elif tag.startswith('V'):
            this_ref_start = -1

    for start, stop in references_found:
        yield (" ".join(LanguageModelBase.normalize_title('', x[0])
                        for x in tags[max(0, start - padding):start]),
               " ".join(LanguageModelBase.normalize_title('', x[0])
                        for x in tags[start:stop + 1]),
               " ".join(LanguageModelBase.normalize_title('', x[0])
                        for x in tags[stop + 1:stop + padding + 1]))

def build_lm_data(path, output):
    cw = CachedWikipedia(path, "")
    o = open(output, 'w')

    count = 0
    for i in [x.split("/")[-1] for x in glob("%s/*" % path)]:
        count += 1
        if count % 1000 == 0:
            print("%i\t%s" % (count, i))
        page = cw[i]

        for ss in nltk.sent_tokenize(page.content):
            o.write("%s\n" % " ".join(kTOKENIZER(ss.lower())))

class Mentions(AbstractFeatureExtractor):
    def __init__(self):
        super().__init__()
        question_db = QuestionDatabase(QB_QUESTION_DB)
        answers = set(x for x, y in text_iterator(
            False, "", False, question_db, False, "", limit=-1, min_pages=MIN_APPEARANCES))
        self.answers = answers
        self.initialized = False
        self.refex_count = defaultdict(int)
        self.refex_lookup = defaultdict(set)
        self._lm = None
        self.generate_refexs(self.answers)
        self.pre = []
        self.ment = []
        self.suf = []
        self.text = ""
        self.kenlm_path = data_path(KEN_LM)

    @property
    def name(self):
        return 'mentions'

    @property
    def lm(self):
        if not self.initialized:
            self._lm = kenlm.LanguageModel(self.kenlm_path)
            self.initialized = True
        return self._lm

    def score_guesses(self, guesses, text):
        # Find mentions if the text has changed
        for guess in guesses:
            if text != self.text:
                self.text = text
                self.pre = []
                self.ment = []
                self.suf = []
                # Find prefixes, suffixes, and mentions
                for pp, mm, ss in find_references(text):
                    # Exclude too short mentions
                    if len(mm.strip()) > 3:
                        self.pre.append(pp.lower())
                        self.suf.append(ss.lower())
                        self.ment.append(mm.lower())

            best_score = float("-inf")
            for ref in self.referring_exs(guess):
                for pp, ss in zip(self.pre, self.suf):
                    pre_tokens = kTOKENIZER(pp)
                    ref_tokens = kTOKENIZER(ref)
                    suf_tokens = kTOKENIZER(ss)

                    query_len = len(pre_tokens) + len(ref_tokens) + len(suf_tokens)
                    query = " ".join(pre_tokens + ref_tokens + suf_tokens)
                    score = self.lm.score(query)
                    if score > best_score:
                        best_score = score / float(query_len)
            if best_score > float("-inf"):
                res = "|%s score:%f" % (self.name, best_score)
            else:
                res = "|%s missing:1" % self.name

            norm_title = LanguageModelBase.normalize_title('', guess)
            assert ":" not in norm_title
            for mm in self.ment:
                assert ":" not in mm
                res += " "
                res += ("%s~%s" % (norm_title, mm)).replace(" ", "_")

            yield res

    def generate_refexs(self, answer_list):
        """
        Given all of the possible answers, generate the referring expressions to
        store in dictionary.
        """

        # TODO: Make referring expression data-driven

        for aa in answer_list:
            ans = aa.split("_(")[0]
            for jj in ans.split():
                # each word and plural form of each word
                self.refex_lookup[aa].add(jj.lower())
                self.refex_lookup[aa].add(pluralize(jj).lower())
                self.refex_count[jj] += 1
                self.refex_count[pluralize(jj)] += 1

            # answer and plural form
            self.refex_count[ans.lower()] += 1
            self.refex_count[pluralize(ans).lower()] += 1
            self.refex_lookup[aa].add(ans.lower())
            self.refex_lookup[aa].add(pluralize(ans).lower())

            # THE answer
            self.refex_count["the %s" % ans.lower()] += 1
            self.refex_lookup[aa].add("the %s" % ans.lower())

    def referring_exs(self, answer, max_count=5):
        """

        Given a Wikipedia page, generate all of the referring expressions.
        Right now just rule-based, but should be improved.
        """
        for ii in self.refex_lookup[answer]:
            if self.refex_count[ii] < max_count:
                yield ii


import argparse
import os

import dash
import dash_core_components as dcc
import dash_html_components as html
from dash.dependencies import Input, Output

from datasetinsights.visualization.object_detection import (
    render_object_detection_layout,
)
from datasetinsights.visualization.overview import overview

this_dir = os.path.dirname(os.path.abspath(__file__))
css_file = os.path.join(this_dir, "stylesheet.css")

app = dash.Dash(__name__, external_stylesheets=[css_file],)
app.config.suppress_callback_exceptions = True

def main_layout(data_root):
    """ Method for generating main app layout.

    Args:
        data_root(str): path to the dataset.

    Returns:
        html layout: main layout design with tabs for overview statistics
            and object detection.
    """
    app_layout = html.Div(
        [
            html.H1(
                children="Dataset Insights",
                style={
                    "textAlign": "center",
                    "padding": 20,
                    "background": "lightgrey",
                },
            ),
            html.Div(
                [
                    dcc.Tabs(
                        id="page_tabs",
                        value="dataset_overview",
                        children=[
                            dcc.Tab(
                                label="Overview", value="dataset_overview",
                            ),
                            dcc.Tab(
                                label="Object Detection",
                                value="object_detection",
                            ),
                        ],
                    ),
                    html.Div(id="main_page_tabs"),
                ]
            ),
        ]
    )
    return app_layout

@app.callback(
    Output("main_page_tabs", "children"), [Input("page_tabs", "value")]
)
def render_content(tab):
    if tab == "dataset_overview":
        return overview(data_root)
    elif tab == "object_detection":
        return render_object_detection_layout(data_root)

def check_path(path):
    if os.path.isdir(path):
        return path
    else:
        raise ValueError(f"Path {path} not found")

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--data-root", help="Path to the data root")
    args = parser.parse_args()
    data_root = check_path(args.data_root)
    app.layout = main_layout(data_root)
    app.run_server(debug=True)


from django.conf.urls import patterns, include, url

__author__ = 'Stanislav Filin'

urlpatterns = patterns('',
                       url(r'^login/', 'loginsys.views.login'),
                       url(r'^logout/', 'loginsys.views.logout'),
                       url(r'^register/', 'loginsys.views.register'),
                       url(r'^profile/', 'loginsys.views.profile'),
                       url(r'^experiment/', 'loginsys.views.experiment'),
                       )


import sys, json, requests, datetime
try:
	import sh
	shell=True
except:
	shell=False

try:
	HGR_ID = sys.argv[1]
except IndexError:
	HGR_ID = input("HGR ID: ")

try:
	GEO_ID = int(sys.argv[2])
except IndexError:
	GEO_ID = int(input("Geonames ID: "))

dataset_file = "output/enhanced_dataset.json"
with open(dataset_file,'r',encoding='utf-8') as fp:
	dataset = json.load(fp)

dataset[HGR_ID]['geo'] = {}

R = requests.get("http://api.geonames.org/getJSON?geonameId={}&username=jaguillette".format(GEO_ID))
geoinfo = R.json()

dataset[HGR_ID]['geo']['geonameId'] = "http://www.geonames.org/{}".format(GEO_ID)
dataset[HGR_ID]['geo']['pres_country'] = geoinfo['countryName']
dataset[HGR_ID]['geo']['x_coord'] = geoinfo['lng']
dataset[HGR_ID]['geo']['y_coord'] = geoinfo['lat']
dataset[HGR_ID]['geo']['coded'] = "manually"
dataset[HGR_ID]['geo']['country_code'] = geoinfo['countryCode']
dataset[HGR_ID]['geo']['pres_loc'] = "{}, {}".format(geoinfo['name'],geoinfo['adminName1'])
dataset[HGR_ID]['query_datetime'] = datetime.datetime.strftime(datetime.datetime.now(),"%Y-%m-%dT%H:%M:%S")

with open(dataset_file,'w',encoding='utf-8') as fp:
	json.dump(dataset,fp,sort_keys=True,indent=2)

if shell:
	git = sh.git.bake(_cwd='/home/jeremy/Documents/pro/fung_library/gazetteer/fungHGR/')
	print(git.add(dataset_file))
	print(git.commit(m="Added {} to dataset after confirming it corresponds to GeoNames ID {}".format(dataset[HGR_ID]['name_old_orth'],GEO_ID)))

#print("Georeferencing successfully added for {}.\nLocation currently corresponds to modern location {}, {}.".format(dataset[HGR_ID]['ru_old_orth'],dataset[HGR_ID]['geo']['pres_loc'],dataset[HGR_ID]['geo']['pres_country']))


from flask import Flask, render_template, request, redirect, Blueprint, url_for
from flask_login import current_user
from werkzeug.utils import secure_filename
from app import app
from helpers import *
from models.base_model import BaseModel
from models.user import User
from models.image import Image
import datetime

images_blueprint = Blueprint('images',
                            __name__,
                            template_folder='templates/images')

ALLOWED_EXTENSIONS = set(['txt', 'pdf', 'png', 'jpg', 'jpeg', 'gif'])

def allowed_file(filename):
    return '.' in filename and \
           filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS

# to create new post/upload images
def create_gallery_image(output):
    gallery_pic = Image(user_id=current_user.id, image_name = output)
    gallery_pic.save()

# to update profile image
def update_profile_pic(output):
    pic = User.update(profile_image_name = output).where(User.id == current_user.id)
    pic.execute()

# to send the file from the user’s computer directly to the bucket
@images_blueprint.route("/<id>", methods=["POST"])
def upload_file(id):
	# A
    if "user_file" not in request.files and "image_file" not in request.files:
        return "No file in request.files"

	# B If the key is in the object, we save it in a variable called file.
    if "user_file" in request.files:
        file = request.files["user_file"]
    if "image_file" in request.files:
        file = request.files["image_file"]
    """
        These attributes are also available
        file.filename               # The actual name of the file
        file.content_type
        file.content_length
        file.mimetype
    """
	# C. We check the filename attribute on the object and if it’s empty,
    # it means the user sumbmitted an empty form, so we return an error message.
    if file.filename == "":
        return "Please select a file"

	# D. we check that there is a file and that it has an allowed filetype
    if file and allowed_file(file.filename):
        file.filename = secure_filename(file.filename)
        output = upload_file_to_s3(file, app.config['S3_BUCKET'])
        if "user_file" in request.files:
            update_profile_pic(output)
            return redirect(url_for("users.edit", id=id))
        if "image_file" in request.files:
            create_gallery_image(output)
            return redirect(url_for("users.show", username=current_user.username))
    else:
        return redirect("/<id>")

# user.profile_image_name # stored in database but returns just the path without domain

# function to upload a file to S3
def upload_file_to_s3(file, bucket_name, acl="public-read"):

    """
    Docs: http://boto3.readthedocs.io/en/latest/guide/s3.html
    """
    u_name = secure_filename(str(current_user.id) + file.filename + str(datetime.datetime.now()))

    try:

        s3.upload_fileobj(
            file,
            bucket_name,
            u_name,
            ExtraArgs={
                "ACL": acl,
                "ContentType": file.content_type
            }
        )

    except Exception as e:
        # This is a catch all exception, edit this part to fit your needs.
        print("Something Happened: ", e)
        return e
    return u_name


from __future__ import print_function, division

__author__ = 'Amrit, Zhe, Jerry, Jack'

from Tkinter import *
import os
from func_GUI import *
import re
import random
from email import message_from_string

# BASE_DIR = os.path.dirname(os.path.abspath(__file__))
Mails_directory = "../Mails/initial2.txt"

"convert the original email to string list"

def email_parser(mailText):
    content = message_from_string(mailText)  # extract email content totally
    subject = content['subject']  # parse for email subject
    body = []
    if content.is_multipart():  # parse for email body
        for payload in content.get_payload():
            body.append(payload.get_payload())
    else:
        body.append(content.get_payload())
    body_segments = re.sub(r"\n|(\\(.*?){)|}|[!$%^&*#()_+|~\-={}\[\]:\";'<>?,.\/\\]|[0-9]|[@]", '',
                           ''.join(body))  # filter characters for email body
    body_keywords = re.sub('\s+', ' ', body_segments)
    subject_segments = re.sub(r"\n|(\\(.*?){)|}|[!$%^&*#()_+|~\-={}\[\]:\";'<>?,.\/\\]|[0-9]|[@]", '',
                              ''.join(subject))  # filter characters for email subject
    subject_keywords = re.sub('\s+', ' ', subject_segments)
    mail_subject = subject_keywords.lower()
    mail_body = body_keywords.lower()
    return subject, body, mail_subject, mail_body  # return original subject, body, and processed subject and body

"read the initial setup from initial.txt"

def readfile(filename):
    global my_folder, pool, list_mails
    with open(filename, 'r') as f:
        for doc in f.readlines():
            try:
                mailText=''
                with open("../"+doc.strip(), 'r') as fd:
                    for docc in fd.readlines():
                        mailText+=docc
                subject, body, mail_subject, mail_body = email_parser(mailText)
                true_label = get_true_label(mailText).lower()
                mail_body = mail_subject + " "+mail_body
                newmail = Email(mail_body)
                newmail.set_read()
                newmail.raw_email = body[0]
                newmail.subject=subject
                newmail.trueLabel = true_label
                newmail.set_folder(true_label)
                newmail.set_label(true_label)
                newmail.credit=1
                if not newmail.label in my_folder.names:
                    my_folder.addfolder(newmail.label)
                pool.append(newmail)
            except:
                pass
    my_folder.train(pool)
    list_mails = pool

def readnew():
    with open("../Mails/newmails.txt", 'r') as f:
        for doc in f.readlines():
            new_mails.append(doc.strip())

def get_true_label(raw_email):
    for line in raw_email.split('\r\n'):
        if line.startswith('X-Folder:'):
            folder = line.split('X-Folder:')[1]
            true_lable = folder.split('\\')[-1]
            print ('folder: ' + true_lable)
            return true_lable

"when new email comes in"
def new_email(mailText):
    global list_mails, my_folder
    subject, body, mail_subject, mail_body = email_parser(mailText)
    true_label = get_true_label(mailText).lower()
    mail_body = mail_subject + " "+mail_body
    newmail = Email(mail_body)
    newmail.raw_email = body[0]
    newmail.trueLabel = true_label
    newmail.subject=subject
    list_mails.append(newmail)
    my_folder.predict(newmail)

class Application(Frame):

    def refresh(self):
        global list_mails, currentfolder, my_folder
        self.unread.delete(first=0, last=self.unread.size())
        self.read.delete(first=0, last=self.read.size())
        folder_unreadNum = {foldername: 0 for foldername in my_folder.names}
        for i, mail in enumerate(list_mails):
            # Count the unread emails for each folder.
            for f in mail.folder:
                if f not in folder_unreadNum:
                    folder_unreadNum[f] = 0
                if not mail.read:
                    folder_unreadNum[f] += 1
            ## update Read and Unread Box Widget
            if currentfolder in mail.folder:
                if mail.read:
                    self.read.insert('end', "%0.4d : %s" % (i, mail.subject))
                else:
                    self.unread.insert('end', "%0.4d : %s" % (i, mail.subject))
        for folder, button in self.folderName_button.iteritems():
            # if folder in ['uncertain', 'trash']:
            #     continue
            unreadNum = folder_unreadNum[folder]
            if unreadNum == 0:
                button.config(text=folder)
                button.config(bg='White')
            else:
                button.config(text=folder + ' (' + str(unreadNum) + ')' )

    def button_command(self, a):
        global currentfolder
        currentfolder = a
        self.refresh()

    # def del_un(self):
    #     x= int(self.unread.get(ANCHOR).split(' : ')[0])
    #     self.unread.delete(ANCHOR)
    #     for i,k in enumerate(list_labels_mails[x]):
    #         if k == "trash":
    #             list_labels_mails[x][i] = ""
    #         if k == "inbox":
    #             list_labels_mails[x][i] = "trash"
    #         if k == "uncertain":
    #             list_labels_mails[x][i] = "trash"
    #
    # def del_re(self):
    #     x= int(self.read.get(ANCHOR).split(' : ')[0])
    #     self.read.delete(ANCHOR)
    #     for i,k in enumerate(list_labels_mails[x]):
    #         if k == "trash":
    #             list_labels_mails[x][i] = ""
    #         if k == "inbox":
    #             list_labels_mails[x][i] = "trash"
    #         if k == "uncertain":
    #             list_labels_mails[x][i] = "trash"

    def read_user(self, event):
        global list_mails, currentfolder
        w = event.widget
        email_id = w.get(w.curselection()).split(' : ')[0]
        email = list_mails[int(email_id)]
        true_label = email.get_trueLabel()
        if email.get_raw() == '':
            pop_message = email.get_body()
        else:
            pop_message = '*** '+ true_label + ' *** \n' + email.get_raw()[:500]
        self.popup1(pop_message)
        if not (currentfolder == 'uncertain' or currentfolder == 'trash'):
            activity_yes(list_mails[int(w.get(w.curselection()).split(' : ')[0])], currentfolder)
        self.refresh()

    def showscore(self):
        global my_folder
        message1="Precision: %f" %my_folder.P
        message2="Recall: %f" %my_folder.R
        message3="F1: %f" %my_folder.F1
        print("Precision: %f" %my_folder.P)
        print("Recall: %f" %my_folder.R)
        print("F1: %f" %my_folder.F1)
        self.popup1(message1+'\n'+message2+'\n'+message3)

    def incoming(self):
        try:
            x=random.randint(0,len(new_mails)-1)
            l=''
            with open("../"+new_mails.pop(x), 'r') as f:
                for doc in f.readlines():
                    l+=doc
            new_email(l)

            self.refresh()
        except:
            self.showscore()

    def unread_user(self, event):
        # w = event.widget
        # self.popup1(w.get(w.curselection()).split(' : ')[1])
        # global list_mails, currentfolder
        global list_mails, currentfolder
        w = event.widget
        email_id = w.get(w.curselection()).split(' : ')[0]
        email = list_mails[int(email_id)]
        true_label = email.get_trueLabel()
        if email.get_raw() == '':
            pop_message = email.get_body()
        else:
            pop_message = '*** '+ true_label + ' *** \n' + email.get_raw()[:500]
        self.popup1(pop_message)

        list_mails[int(w.get(w.curselection()).split(' : ')[0])].set_read()
        if not (currentfolder == 'uncertain' or currentfolder == 'trash'):
            activity_yes(list_mails[int(w.get(w.curselection()).split(' : ')[0])], currentfolder)
        self.refresh()

    def mov_command(self, targetfolder):
        global list_mails, currentfolder
        list_mails[int(self.waste)].folder = [targetfolder]
        if not (targetfolder == 'uncertain' or targetfolder == 'trash'):
            activity_no(list_mails[int(self.waste)], targetfolder)
        self.refresh()

    def c_labels_command(self):
        self.popup()
        if (self.entryValue() != ''):
            a = self.entryValue()
            global my_folder
            my_folder.addfolder(a)
            self.create_folder(a)

    def create_folder(self, name):
        self.buttons.append(name)
        x1 = Button(self.m, text=name, fg="red", command=lambda name=name: self.button_command(name), state=ACTIVE)
        self.m.add(x1)
        self.folderName_button[name] = x1
        self.count += 1

    def change_checkbox(self, bv, id):
        now = bv.get()
        self.checkbox[id] = now
        # print (now)
        features[id] = now
        print(features)

    def create_checkbox(self, id, default=True):
        bv = BooleanVar()
        bv.set(default)
        bv.trace("w", lambda name, index, mode, bv=bv: self.change_checkbox(bv, id))
        self.checkbox[id] = bv
        c = Checkbutton(self.m3, text=feature_names[id], variable=bv)
        self.m3.add(c)

    def createWidgets(self):
        self.m = PanedWindow(self, orient=VERTICAL)
        self.m.pack(side=LEFT, expand=1)
        for name in my_folder.names:
            self.create_folder(name)
        # self.USER.pack(side=BOTTOM, anchor=W, fill=X)
        self.m2 = PanedWindow(self, orient=VERTICAL)
        self.m2.pack(fill=BOTH, side=LEFT, expand=1)
        # self.m.add(self.m2)
        # New incoming mail

        self.top_right = Label(self.m2, text="Unread")
        self.m2.add(self.top_right)

        self.unread = Listbox(self.m2, width=120, height=20)
        self.unread.grid(column=0, row=0, sticky='nwes')
        self.s = Scrollbar(command=self.unread.yview, orient='vertical')
        # self.s.grid(column=0, row=0, sticky="ns")
        self.unread.config(yscrollcommand=self.s.set)
        self.unread.insert('end', "Unread Mails")
        self.unread.bind("<Double-Button-1>", self.unread_user)
        self.unread.bind("<Button-2>", self.popup_menu)
        self.unread.pack()
        self.m2.add(self.unread)

        self.bottom_right = Label(self.m2, text="Read")
        self.m2.add(self.bottom_right)
        self.read = Listbox(self.m2, width=120, height=20)
        self.read.grid(column=0, row=0, sticky='nwes')
        self.s = Scrollbar(command=self.read.yview, orient='vertical')
        # self.s.grid(column=0, row=0, sticky="ns")
        self.read.config(yscrollcommand=self.s.set)
        self.read.insert('end', "Read Mails")
        self.read.bind("<Double-Button-1>", self.read_user)
        self.read.bind("<Button-2>", self.popup_menu)

        self.read.pack()
        self.m2.add(self.read)

        self.m3 = PanedWindow(self, orient=VERTICAL)
        self.m3.pack(side=RIGHT, expand=1)
        # self.delete_un = Button(self, text="DELETE UNREAD", command=self.del_un)
        # self.delete_re = Button(self, text="DELETE READ", command=self.del_re)
        self.CREATE_Labels = Button(self, text="CREATE_LABELS", fg="red", command=self.c_labels_command, state=ACTIVE)
        self.newm = Button(self, text="New Mail", fg="red", command=self.incoming, state=ACTIVE)
        self.show = Button(self, text="Show Score", fg="red", command=self.showscore, state=ACTIVE)
        self.m3.add(self.newm)
        self.m3.add(self.CREATE_Labels)
        self.m3.add(self.show)

        for feature_id in range(len(features)):
            self.create_checkbox(feature_id)

        self.button_command('uncertain')

    def popup_menu(self, event):
        w = event.widget
        self.aMenu = Menu(self, tearoff=0)
        # print(labels_gui)
        for i in my_folder.names:
            if i != 'uncertain' and i != 'trash':
                self.aMenu.add_command(label=i, command=lambda i=i: self.mov_command(i.lower()))
        self.waste = w.get(w.curselection()).split(' : ')[0]
        self.aMenu.post(event.x_root, event.y_root)

    def popup(self):
        self.w = popupWindow(self.Frame)
        self.Frame.wait_window(self.w.top)

    def popup1(self, data):
        self.w = popupWindow1(self.Frame, data)
        self.Frame.wait_window(self.w.top)

    def entryValue(self):
        return self.w.value

    def __init__(self, master=None):
        Frame.__init__(self, master)
        self.Frame = master
        self.count = 0
        self.buttons = []
        self.folderName_button = {}
        self.checkbox = {}
        self.pack()
        self.waste = ''
        self.createWidgets()
        # print(Application.__getitem__(self, Button)

class popupWindow1(object):
    def __init__(self, master, data):
        self.data = data
        top = self.top = Toplevel(master)
        self.l = Label(top, text=self.data)
        self.l.pack()
        self.b = Button(top, text='Ok', command=self.cleanup)
        self.b.pack()

    def cleanup(self):
        self.top.destroy()

class popupWindow(object):
    def __init__(self, master):
        top = self.top = Toplevel(master)
        self.l = Label(top, text="Label Creation")
        self.l.pack()
        self.e = Entry(top)
        self.e.pack()
        self.b = Button(top, text='Ok', command=self.cleanup)
        self.b.pack()

    def cleanup(self):
        self.value = self.e.get()
        self.top.destroy()

"format email, str to csr"

def format(email):
    global hashemail
    x = vsm([email])
    x = hashemail(x)
    x = l2normalize(x)
    return x

class Folder(object):
    def __init__(self, thres=0.35):
        self.names = ['uncertain', 'trash']
        self.classifier = []
        self.thres = thres
        self.num = 0
        self.T={}
        self.F={}
        self.P=0
        self.R=0
        self.F1=0
        self.FR={}

    def addfolder(self, name):
        self.names.append(name)
        self.num = self.num + 1
        self.T[name]=0
        self.F[name]=0
        self.FR[name]=0

    def update(self):
        folders=self.names[2:]
        self.P=0
        self.R=0
        self.F1=0
        if folders:
            for fold in folders:
                if self.T[fold]==0:
                    self.P=self.P+0
                    self.R=self.R+0
                else:
                    self.P=self.P+self.T[fold]/(self.T[fold]+self.F[fold])
                    self.R=self.R+self.T[fold]/(self.T[fold]+self.FR[fold])
            self.P=self.P/len(folders)
            self.R=self.R/len(folders)
            if self.P and self.R:
                self.F1=self.P*self.R*2/(self.P+self.R)
            else:
                self.F1=0

    "call this to train the classifier"

    def train(self, training_set):
        labels = []
        bodies = []
        for mail in training_set:
            labels.append(mail.label)
            tmp = mail.mat
            if not bodies == []:
                bodies = csr_vstack(bodies, tmp)
            else:
                bodies = tmp
        self.classifier = do_SVM(labels, bodies)

    "return the list of folder names the email would belong to, also will change mail.folder"

    def predict(self, mail):
        global features
        proba = self.classifier.predict_proba(mail.mat)[0]
        mail.folder = []
        for ind, label in enumerate(self.classifier.classes_):
            mail.proba[label] = proba[ind]
            if proba[ind] > self.thres:
                mail.folder.append(label)
        if not mail.folder:
            mail.folder = ['uncertain']
        if not features[2]:
            if len(mail.folder) > 1:
                mail.folder = [self.classifier.classes_[np.argmax(proba)]]

        for fold in mail.folder:
            if fold == mail.trueLabel:
                try:
                    self.T[fold]=self.T[fold]+1
                except:
                    self.T[fold]=1
            else:
                try:
                    self.F[fold]=self.F[fold]+1
                except:
                    self.F[fold]=1
        if mail.trueLabel not in mail.folder:
            try:
                self.FR[mail.trueLabel]=self.FR[mail.trueLabel]+1
            except:
                self.FR[mail.trueLabel]=1
        self.update()

        return mail.folder

class Email(object):
    def __init__(self, body=""):
        self.body = body
        if body:
            self.mat = format(body)
        else:
            self.mat = []
        self.folder = []
        self.label = ''
        self.trueLabel = ''
        self.credit = 0
        self.proba = {}
        self.read = False
        self.raw_email = ''
        self.subject=""

    def set_read(self):
        self.read = True

    def set_label(self, label):
        self.label = label

    def set_body(self, body):
        self.body = body
        self.mat = format(body)

    def set_credit(self, credit):
        self.credit = credit

    def set_folder(self, foldername):
        self.folder = [foldername]

    def get_body(self):
        return self.body

    def get_raw(self):
        return self.raw_email

    def get_probab(self):
        return self.proba

    def get_label(self):
        return self.label

    def get_trueLabel(self):
        return self.trueLabel

    def get_credit(self):
        return self.credit

"If user read, forward, reply this email. Will not trigger if current_folder is uncertain or trash"

def activity_yes(email, current_folder):
    global pool, features
    if features[0]:
        email.set_label(current_folder)
        try:
            email.set_credit(1 - email.proba[current_folder])
        except:
            pass
        if email not in pool:
            pool.append(email)
    check_credit()

"If user move this email into a target_folder. Will not trigger if target_folder is uncertain or trash."

def activity_no(email, target_folder):
    global pool, features
    if features[0] or features[1]:
        email.set_label(target_folder)
        try:
            email.set_credit(1 - email.proba[target_folder])
        except:
            email.set_credit(1)
        if email not in pool:
            pool.append(email)
    check_credit()

"Runs every mid-night? Retrain the classifier or not"

def check_credit():
    global pool, my_folder, saturation, lastcan
    can = []
    for label in my_folder.names:
        if label=='uncertain' or label=='trash': continue
        collect = [x for x in pool if x.label == label]
        credits = [x.credit for x in collect]
        many=int(len(pool)/(len(my_folder.names)-2))
        can.extend(list(np.array(collect)[np.argsort(credits)[-many:]]))
    if can!=lastcan:
        my_folder.train(can)
        lastcan=can
    if len(pool) > saturation:
        pool=can
    return True

if __name__ == '__main__':
    global feature_number, hashemail, my_folder, pool, saturation, currentfolder, features, feature_names, new_mails, list_mails, lastcan

    "Global variables, need to be initialized"
    feature_number = 4000
    hashemail = hasher(feature_number)
    my_folder = Folder()
    pool = []
    list_mails=[]
    saturation = 1000
    currentfolder = "uncertain"
    features = [True, True, True]
    feature_names = ['implicit feedback', 'explicit feedback', 'multi-folder']
    "feature[0]: implicit user feedback; feature[1]: explicit user feedback; feature[2]: multi-folder"
    new_mails=[]
    lastcan=[]

    #new mail reading
    readnew()

    # pool of mails with intial labels.
    readfile(Mails_directory)
    # check_credit()
    # GUI
    root = Tk()
    root.title("Mailbox")
    root.minsize(width=1000, height=600)
    app = Application(master=root)
    app.mainloop()
    root.destroy()


import _io
from django import forms
from file_resubmit.admin import AdminResubmitImageWidget
from PIL import Image
from note.models import Note
from smedialink.settings import MEDIA_ROOT

class NoteCreationForm(forms.ModelForm):
    class Meta:
        model=Note
        fields=('text','image',)

    image = forms.ImageField(
        required=False,
        label='Изображение',
        widget=AdminResubmitImageWidget(
            attrs={'class': 'form-control', 'data-validation': 'mime size', 'data-validation-allowing': 'jpg, png, gif',
                   'data-validation-max-size': '20Mb',
                   'data-validation-error-msg-size': 'Вы не можете загружать изображения больше 20Мб',
                   'data-validation-error-msg-mime': 'Вы можете загружать только изображения.'}
        )
    )

    text = forms.CharField(
        label="Текст",
        required=False,
        widget=forms.Textarea(
            attrs={'class': 'form-control', 'rows': '6'}
        )
    )

    def clean_image(self):
        correct_width,correct_height=True,True
        start_img=self.cleaned_data['image']
        if not start_img is None:
            img = Image.open(start_img)
            width, height = img.size
            if not (320<=width<=2048):
                correct_width=False
            if not (240<=height<=1968):
                correct_height=False
            if correct_height and correct_width:
                return start_img
            elif correct_height==False and correct_width:
                fheight=int((width*3)/4)
                if (240<=fheight<=1968):
                    img.resize((width, fheight), Image.ANTIALIAS)
                    return start_img
                else:
                    raise forms.ValidationError("Форамат изображения должен быть от 320х240 до 2048х1968")
            # elif correct_height and correct_width==False:
            #     pass
            else:
                raise forms.ValidationError("Форамат изображения должен быть от 320х240 до 2048х1968")

    def save(self, commit=True,pil=None):
        note = super(NoteCreationForm, self).save(commit=False)
        if commit:
            note.save()
        return note

class SentNoteForm (forms.Form):

    recipient_email = forms.EmailField(
        label='Email получателя',
        widget=forms.TextInput(
            attrs={'placeholder': 'Email получателя', 'class': 'form-control', 'data-validation': 'email'}),
    )


import asyncio
import hashlib
import time

import pytest
from async_generator import async_generator, yield_

import aiomisc

class HashServer(aiomisc.service.TCPServer):
    async def handle_client(
        self, reader: asyncio.StreamReader,
        writer: asyncio.StreamWriter
    ):
        hasher = hashlib.md5()

        chunk = await reader.read(65534)
        while chunk:
            hasher.update(chunk)
            writer.write(hasher.digest())
            chunk = await reader.read(65534)

        writer.close()
        await writer.wait_closed()

@pytest.fixture()
def server_port(aiomisc_unused_port_factory) -> int:
    return aiomisc_unused_port_factory()

@pytest.fixture()
def service(
    loop: asyncio.AbstractEventLoop,
    server_port, localhost,
) -> HashServer:
    return HashServer(port=server_port, address=localhost)

@pytest.fixture()
def services(service: HashServer):
    return [service]

@pytest.fixture()
@async_generator
async def proxy(tcp_proxy, localhost, server_port):
    async with tcp_proxy(localhost, server_port, buffered=True) as proxy:
        await yield_(proxy)

async def test_proxy_client(proxy):
    reader, writer = await proxy.create_client()
    payload = b"Hello world"

    writer.write(payload)
    hash = await asyncio.wait_for(reader.readexactly(16), timeout=1)

    assert hash == hashlib.md5(payload).digest()

async def test_proxy_client_close(proxy):
    reader, writer = await proxy.create_client()
    payload = b"Hello world"

    writer.write(payload)
    hash = await asyncio.wait_for(reader.readexactly(16), timeout=1)

    assert hash == hashlib.md5(payload).digest()

    assert not reader.at_eof()
    await proxy.disconnect_all()

    assert await asyncio.wait_for(reader.read(), timeout=1) == b""
    assert reader.at_eof()

async def test_proxy_client_slow(proxy):
    delay = 0.1
    proxy.set_delay(delay, delay)

    reader, writer = await proxy.create_client()
    payload = b"Hello world"

    delta = -time.monotonic()
    writer.write(payload)
    hash = await asyncio.wait_for(reader.readexactly(16), timeout=2)
    delta += time.monotonic()

    assert delta >= delay, [c.delay for c in proxy.clients]

    assert hash == hashlib.md5(payload).digest()

async def test_proxy_client_with_processor(proxy):
    processed_request = b"Never say hello"

    proxy.set_content_processors(
        lambda _: processed_request,
        lambda chunk: chunk[::-1],
    )

    reader, writer = await proxy.create_client()
    payload = b"Hello world"

    writer.write(payload)
    hash = await asyncio.wait_for(reader.read(16), timeout=2)

    assert hash == hashlib.md5(processed_request).digest()[::-1]


# -*- coding: utf-8 -*-

import os

from flask import render_template, redirect, url_for, request, g
from flask import send_from_directory
from flask_login import login_user, logout_user, current_user, login_required

from app import app, login_manager
from app import db
from .models import User, Party

@login_manager.user_loader
def load_user(user_id):
    return User.query.get(int(user_id))

def validateAndAdd(id):
    cur_user = User.query.filter_by(id_number=current_user.id_number).all()
    party = Party.query.filter_by(id=id).first()
    party.vote_count += 1
    for user in cur_user:
        user.voted = True
    db.session.commit()

@app.route('/', methods=['GET'])
@app.route('/index', methods=['GET', 'POST'])
@login_required
def index():
    error = None
    if request.method == 'POST':
        validateAndAdd(request.form['party_name'])
        return redirect(url_for('login'))
    g.user = current_user  # global user parameter used by flask framwork
    parties = Party.query.all()
    return render_template('index.html', title='Home', user=g.user, parties=parties, error=error)

@app.route('/login', methods=['GET', 'POST'])
def login():
    error = None
    if request.method == 'POST':
        ## Validate user
        first_name = request.form['first_name']
        last_name = request.form['last_name']
        id_number = request.form['id_number']
        check_db = User.query.filter_by(id_number=id_number).all()
        if check_db == None:
            error = u'המצביע אינו מופיע בבסיס הנתונים'
        else:
            for user in check_db:
                if first_name == user.first_name and last_name == user.last_name and id_number == user.id_number:
                    if user.voted:
                        error = u'ניתן להצביע פעם אחת בלבד'
                        return render_template('login.html', error=error)

                    else:
                        # user = User.query.filter_by(id_number=id_number).first()
                        login_user(user)  ## built in 'flask login' method that creates a user session
                        return redirect(url_for('index'))

            error = u'המצביע אינו מופיע בבסיס הנתונים'
    if error is not None:
        return render_template('login.html', error=error), 404

    return render_template('login.html', error=error), 200

## will handle the logout request
@app.route('/logout')
@login_required
def logout():
    logout_user()  ## built in 'flask login' method that deletes the user session
    return redirect(url_for('index'))

## secret page that shows the user name
@app.route('/secret', methods=['GET'])
@login_required
def secret():
    return 'This is a secret page. You are logged in as {} {} {}'.format(current_user.first_name,
                                                                         current_user.last_name,
                                                                         current_user.id_number)

## will handle the site icon - bonus 2 points for creative new icon
@app.route('/favicon.ico')
def favicon():
    return send_from_directory(os.path.join(app.root_path, 'static'), 'Realfavicon.ico',
                               mimetype='image/vnd.microsoft.icon')


#!/usr/bin/python
import os
from sqlite3 import connect as sqlconn

inConn = sqlconn('nsdevman.bbtrac.sqlite3')
inCursor = inConn.cursor()
outConn = sqlconn('nsdevman.blank.sqlite3')
outCursor = outConn.cursor()

for inTable, outTable in (('nsaabill_billmember', 'nsaabill_dbbillmember'),
                          ('nsaabill_billmonthcache', 'nsaabill_dbbillmonthcache'),
                          ('nsproj_project', 'nsproj_dbproject'),
                          ('nsroot_attrdatetime', 'nsroot_dbattrdatetime'),
                          ('nsroot_attrinteger', 'nsroot_dbattrinteger'),
                          ('nsroot_attrmember', 'nsroot_dbattrmember'),
                          ('nsroot_attrpositiveinteger', 'nsroot_dbattrpositiveinteger'),
                          ('nsroot_attrstring', 'nsroot_dbattrstring'),
                          ('nsroot_commentcachedtext', 'nsroot_dbcommentcachedtext'),
                          ('nsroot_commenttext', 'nsroot_dbcommenttext'),
                          ('nsroot_entity', 'nsroot_dbentity'),
                          ('nsroot_idscope', 'nsroot_dbidscope'),
                          ('nsroot_idvalue', 'nsroot_dbidvalue'),
                          ('nsroot_member', 'nsroot_dbmember'),
                          ('nsroot_memberprefs', 'nsroot_dbmemberprefs'),
                          ('nsroot_memberroles', 'nsroot_dbmemberroles'),
                          ('nsroot_perm', 'nsroot_dbperm'),
                          ('nsroot_role', 'nsroot_dbrole'),
                          ('nsroot_roleperms', 'nsroot_dbroleperms'),
                          ('nstool_randombatch', 'nstool_dbrandombatch')):

    rows = list(inCursor.execute('SELECT * FROM %s' % inTable))
    if rows == []: continue
    print('From %s to %s with %u records' % (inTable, outTable, len(rows)))
    outCursor.execute('DELETE FROM %s' % outTable)
    sql = 'INSERT INTO %s VALUES (%s)' % (outTable, ', '.join(['?'] * len(rows[0])))
    outCursor.executemany(sql, rows)
outConn.commit()


# -*- coding: utf-8 -*-
"""
Created on Sun Aug 19 14:25:20 2018

@author: Hemant Rathore
"""

# 6. Timeseries Analysis using Seasonal ARIMA

## Case Study - Seasonal ARIMA Model for Industrial production of electric and gas utilities
## in the United States, from the years 1985–2018, with our frequency being Monthly.

import os

os.chdir("D:\Data Science Training\Data")

import pandas as pd

import numpy as np

energy_data = pd.read_csv("IPG_Dataset.csv")

energy_data.head(10)

energy_data.describe()

# Plot the TS & Notice trend,seasonality,heteroscedasticity in TS

energy_data.plot(figsize=(12,12))

# Decompose TS

from statsmodels.tsa.seasonal import seasonal_decompose

result = seasonal_decompose(energy_data.PRODUCTION, model='multiplicative',freq=12)

from pylab import rcParams

rcParams['figure.figsize'] = 10,10

result.plot()

## Summary

# Hetro.- yes
# trend- yes
# Seasonality - yes

# 1. Variance is increasing so take log to remove heteroscedasticity

import numpy as np

import pandas as pd

energy = energy_data.PRODUCTION

log_energy = np.log(energy)

log_energy.plot()
energy.plot()

# 2 Detrend TS by differencing

from pandas import Series

d_log_energy = Series.diff(log_energy,periods=1)

d_log_energy.plot()

# 3 remove Seasonality by taking seasonal difference

dd_log_energy = Series.diff(d_log_energy,periods=12)

dd_log_energy.plot(figsize=(10,5))

## Model Selection

# ARIMA(p,d,q)[non seasonal], (p,d,q)[12][seasonal]

#(p,1,q)(p,1,q)[12]

# Analyze ACF (MA) and PACF(AR) to conclude model

dd_log_energy.isna()

dd_log_energy.dropna(inplace=True)

import statsmodels.api as sm
import matplotlib.pyplot as plt

fig = plt.figure(figsize=(10,10))

ax = fig.add_subplot()
fig = sm.graphics.tsa.plot_acf(dd_log_energy, lags=50, ax=ax)

# non seasonal : MA(2) as non seasonal is cutting  off after 2 lags (ignore 0th lag)
# seasonal : MA(2) cutting off after 2 lags

#(p,1,2)(p,1,2)[12]

fig = plt.figure(figsize=(10,10))
ax = fig.add_subplot()
fig = sm.graphics.tsa.plot_pacf(dd_log_energy, lags=50, ax=ax)

# non seasonal : AR(2) as cutting off after 2 lags
# seasonal : AR(0) because its tailing off

#(2,1,2)(0,1,2)[12]

arima_model = sm.tsa.statespace.SARIMAX(energy,
                                            order=(2,1,2),
                                            seasonal_order=(0,1,2,12),
                                            enforce_stationarity=False,
                                            enforce_invertibility=False)

arima_model_fit = arima_model.fit()

arima_model_fit.save

pred = arima_model_fit.get_prediction(start = 900, end = 967, dynamic=False)

ax = energy.plot(label='observed')
pred.predicted_mean.plot(ax=ax, label='forecast', alpha=.7)

plt.legend()

ax = energy[850:1000].plot(label='observed',figsize=(15,10),color='red')
pred.predicted_mean.plot(ax=ax, label='forecast', alpha=.7,color='blue')

plt.legend()

Topic13/Cognixia - Data Science with Python - Machine Learning Case Study 6.py


'''
Script: qgis_frontend_test_script.py
Repository: https://github.com/KRS-dev/proeven_verzameling
Author: Kevin Schuurman
E-mail: kevinschuurman98@gmail.com
Summary: Frontend test script to use without qgis
'''
'''
Inputs:
hoogte selectie voor ALLE monsters Zbot en Ztop
Proef types voor triaxiaal proeven: ['CU','UU', 'CD']
Volume gewicht voor triaxiaal proeven Vgmin Vgmax
Rek ea voor selectie bij LST-squares voor triaxiaal proeven
Output file/dir: r'D:\documents\Proeven-Selectie.xlsx'
'''
if __name__ == "__main__":
    hoogte_selectie = [100, -100] # mNAP
    proef_types = ['CU'] # ['CU','CD','UU']
    volume_gewicht_selectie = [9, 22] # kN/m3
    rek_selectie = [5] # %
    output_location = r'D:\Documents\Projects\EXCEL BIS STAT' # Example: D:\Documents\geo_parameters\'
    output_file = 'TRX_Example.xlsx' # Example: TRX_example.xlsx
    show_plot = False # True/False
    save_plot = False # True/False
    loc_ids = [143800, 144566, 144584, 142998, 140980, 48565, 160833, 160838, 160847, 160856, 156721, 156724, 156726, 156728, 138341, 138353, 100652, 140976, 143018, 156720, 198019, 210119, 258134, 270183, 270201, 270518, 270520, 270521, 265149, 262493, 285429, 285431, 285434, 285435, 290032, 297470, 297474, 340699, 340703, 48107, 48563, 48566, 91546, 90312, 107300, 100651, 138342, 138343, 144613]

import sys, os, shutil
# Change the directory so the script can open all files in this directory easily
os.chdir(r'D:\Documents\GitHub\proeven_verzameling')
# Adding the same path to the python system so that qgis_backend can be imported
sys.path.append(r'D:\Documents\GitHub\proeven_verzameling')
import qgis_backend as qb
import pandas as pd
import numpy as np
import openpyxl

# Check if the directory still has to be made.
if os.path.isdir(output_location) == False:
    os.mkdir(output_location)

# Get all meetpunten related to these loc_ids
df_meetp = qb.get_meetpunten(loc_ids)
df_geod = qb.get_geo_dossiers( df_meetp.gds_id )
df_gm = qb.get_geotech_monsters(loc_ids)
df_gm_filt_on_z = qb.select_on_z_coord(df_gm, hoogte_selectie[0], hoogte_selectie[1])
# Add the df_meetp, df_geod and df_gm_filt_on_z to a dataframe dictionary
df_dict = {'BIS_Meetpunten': df_meetp, 'BIS_GEO_Dossiers':df_geod, 'BIS_Geotechnische_Monsters':df_gm_filt_on_z}

df_sdp = qb.get_sdp( df_gm_filt_on_z.gtm_id )
if df_sdp is not None:
    df_sdp_result = qb.get_sdp_result(df_gm.gtm_id)
    df_dict.update({'BIS_SDP_Proeven':df_sdp, 'BIS_SDP_Resultaten':df_sdp_result})

df_trx = qb.get_trx(df_gm_filt_on_z.gtm_id, proef_type = proef_types)
df_trx = qb.select_on_vg(df_trx, volume_gewicht_selectie[1], volume_gewicht_selectie[0])
if df_trx is not None:
    # Get all TRX results, TRX deelproeven and TRX deelproef results
    df_trx_results = qb.get_trx_result(df_trx.gtm_id)
    df_trx_dlp = qb.get_trx_dlp(df_trx.gtm_id)
    df_trx_dlp_result = qb.get_trx_dlp_result(df_trx.gtm_id)
    df_dict.update({'BIS_TRX_Proeven':df_trx, 'BIS_TRX_Results':df_trx_results, 'BIS_TRX_DLP':df_trx_dlp, 'BIS_TRX_DLP_Results': df_trx_dlp_result})

    # Doing statistics on the select TRX proeven
    if len(df_trx.index) > 1:
        ## Create a linear space between de maximal volumetric weight and the minimal volumetric weight
        minvg, maxvg = min(df_trx.volumegewicht_nat), max(df_trx.volumegewicht_nat)
        N = round(len(df_trx.index)/5) + 1
        cutoff = 1 # The interval cant be lower than 1 kn/m3
        if (maxvg-minvg)/N > cutoff:
            Vg_linspace = np.linspace(minvg, maxvg, N)
        else:
            N = round((maxvg-minvg)/cutoff)
            if N < 2:
                Vg_linspace = np.linspace(minvg, maxvg, 2)
            else:
                Vg_linspace = np.linspace(minvg, maxvg, N)

        Vgmax = Vg_linspace[1:]
        Vgmin = Vg_linspace[0:-1]

        df_vg_stat_dict = {}
        df_lst_sqrs_dict = {}
        for ea in rek_selectie:
            ls_list = []
            avg_list = []
            for vg_max, vg_min in zip(Vgmax, Vgmin):
                # Make a selection for this volumetric weight interval
                gtm_ids = qb.select_on_vg(df_trx, Vg_max = vg_max, Vg_min = vg_min, soort = 'nat')['gtm_id']
                if len(gtm_ids) > 0:
                    # Create a tag for this particular volumetric weight interval
                    key = 'Vg: ' + str(round(vg_min,1)) + '-' + str(round(vg_max, 1)) + ' kN/m3'
                    # Get the related TRX results...
                    #
                    ## Potentially the next line could be done without querying the database again
                    ## for the data that is already availabe in the variable df_trx_results
                    ## but I have not found the right type of filter methods in Pandas which
                    ## can replicate the SQL filters
                    #
                    df_trx_results_temp = qb.get_trx_result(gtm_ids)
                    # Calculate the averages and standard deviation of fi and coh for different strain types and add them to a dataframe list
                    mean_fi,std_fi,mean_coh,std_coh,N = qb.get_average_per_ea(df_trx_results_temp, ea)
                    df_avg_temp = pd.DataFrame(index = [key], data = [[vg_min, vg_max, mean_fi, mean_coh, std_fi, std_coh, N]],\
                        columns=['min(Vg)','max(Vg)','mean(fi)','mean(coh)','std(fi)','std(coh)','N'])
                    avg_list.append(df_avg_temp)
                    # Calculate the least squares estimate of the S en T and add them to a dataframe list
                    fi, coh, E, E_per_n, eps, N = qb.get_least_squares(
                        qb.get_trx_dlp_result(gtm_ids),
                        ea = ea,
                        plot_name = 'Least Squares Analysis, ea: ' + str(ea) + '\n' + key,
                        show_plot = show_plot,
                        save_plot = save_plot
                        )
                    df_lst_temp = pd.DataFrame(index = [key], data = [[vg_min, vg_max, fi, coh, E, E_per_n, eps, N]],\
                        columns=['min(Vg)', 'max(Vg)','fi','coh','Abs. Sq. Err.','Abs. Sq. Err./N','Mean Rel. Err. %','N'])
                    ls_list.append(df_lst_temp)
            if len(ls_list)>0:
                df_ls_stat = pd.concat(ls_list)
                df_ls_stat.index.name = 'ea: ' + str(ea) +'%'
                df_lst_sqrs_dict.update({ str(ea) + r'% rek least squares fit':df_ls_stat})
            if len(avg_list)>0:
                df_avg_stat = pd.concat(avg_list)
                df_avg_stat.index.name = 'ea: ' + str(ea) +'%'
                df_vg_stat_dict.update({ str(ea) + r'% rek gemiddelde fit':df_avg_stat})

        df_bbn_stat_dict = {}
        for ea in rek_selectie:
            bbn_list = []
            for bbn_code in pd.unique(df_trx.bbn_kode):
                gtm_ids = df_trx[df_trx.bbn_kode == bbn_code].gtm_id
                if len(gtm_ids > 0):
                    df_trx_results_temp = qb.get_trx_result(gtm_ids)
                    mean_fi,std_fi,mean_coh,std_coh,N = qb.get_average_per_ea(df_trx_results_temp, ea)
                    bbn_list.append(pd.DataFrame(index = [bbn_code], data = [[mean_fi, mean_coh, std_fi, std_coh, N]],\
                        columns=['mean(fi)','mean(coh)','std(fi)','std(coh)','N']))
            if len(bbn_list)>0:
                df_bbn_stat = pd.concat(bbn_list)
                df_bbn_stat.index.name = 'ea: ' + str(ea) +'%'
                df_bbn_stat_dict.update({ str(ea) + r'% rek per BBN code':df_bbn_stat})

# Check if the .xlsx file exists
output_file_dir = os.path.join(output_location, output_file)
if os.path.exists(output_file_dir) == False:
    book = openpyxl.Workbook()
    book.save(output_file_dir)
else:
    name, ext = output_file.split('.')
    i = 1
    while os.path.exists(os.path.join(output_location, name + '{}.'.format(i) + ext)):
        i += 1
    output_file_dir = os.path.join(output_location, name + '{}.'.format(i) + ext)
    book = openpyxl.Workbook()
    book.save(output_file_dir)

# At the end of the 'with' function it closes the excelwriter automatically, even if there was an error
with pd.ExcelWriter(output_file_dir,engine='openpyxl',mode='w') as writer: #writer in append mode so that the NEN tables are kept
    for key in df_dict:
        # Writing every dataframe in the dictionary to a different sheet
        df_dict[key].to_excel(writer, sheet_name = key)

    if df_trx is not None:
        # Write the multiple dataframes of the same statistical analysis for TRX to the same excel sheet by counting rows
        row = 0
        for key in df_vg_stat_dict:
            df_vg_stat_dict[key].to_excel(writer, sheet_name = 'Simpele Vg stat.', startrow = row)
            row = row + len(df_vg_stat_dict[key].index) + 2
        # Repeat...
        row=0
        for key in df_lst_sqrs_dict:
            df_lst_sqrs_dict[key].to_excel(writer, sheet_name = 'Least Squares Vg Stat.', startrow = row)
            row = row + len(df_lst_sqrs_dict[key].index) + 2
        row=0
        for key in df_bbn_stat_dict:
            df_bbn_stat_dict[key].to_excel(writer, sheet_name = 'bbn_kode Stat.', startrow = row)
            row = row + len(df_bbn_stat_dict[key].index) + 2

os.startfile(output_file_dir)


# coding: utf-8
from telethon import TelegramClient, events
from time import sleep
import os, re
from subprocess import getoutput

#id number and api hash for log in the account
api_hash = ''
api_id = ''

#Instance of client
client = TelegramClient('{}/test'.format(os.environ['HOME']), api_id, api_hash)

@client.on(events.NewMessage(chats='PresidenteAMLO'))
async def messageH(event):
    testing = event.raw_text
    if re.search('(?i).?conferencia.*prensa.*matutina.?', testing):
        testing = re.search('https://you.*', testing)
        if testing:
            testing = testing.group(0)
            testing = testing.split()
            testing = testing[0]
            minutes = 0
            while minutes < 180:
                sleep(60)
                minutes += 1
            os.system('youtube-dl -Uicvxf bestaudio -R 50 -o \%\(title\)s\.\%\(ext\)s --write-thumbnail --audio-format mp3 --embed-thumbnail --add-metadata --write-description {}'.format(testing))
            testing = getoutput('youtube-dl -se {}'.format(testing))
            await client.send_file('lamaaudio', '{}/{}.mp3'.format(os.environ['HOME'], testing))

client.start()
client.run_until_disconnected()