import tkinter as tk
from tkinter import CENTER, W, E, S
from tkinter import ttk
from tkinter import messagebox
from tkinter.filedialog import asksaveasfilename, askopenfilename
from tkcalendar import Calendar, DateEntry

from Controller.MainController import get_labs_list, get_customer_list, get_factors_list, generate_protocol, \
    get_measure_list, get_methodologies_list, get_experts_list, get_customer, add_laboratory_click, add_customer_click, \
    add_department_click, add_workplace_click, edit_department_click, edit_workplace_click, export_workplaces_click, \
    import_workplaces_click, import_wp_click

class MainWindow(tk.Frame):
    def __init__(self, root):
        super().__init__(root)
        root.title("Производственный контроль")
        root.geometry("650x445+300+200")
        root.resizable(False, False)
        self.init_main_window()

    def lab_checked(self, event):
        self.combo_cust.configure(values=get_customer_list(), state='normal')

    def customer_checked(self, event):
        self.combo_factor.configure(values=get_factors_list(self.combo_cust.get()), state='normal')
        self.show_button.configure(state='normal')
        self.import_wp_button.configure(state='normal')

    def factor_checked(self, event):
        self.create_button.configure(state='normal')
        self.export_button.configure(state='normal')
        self.import_button.configure(state='normal')

    def show_button_click(self):
        self.generate_tree(self.combo_cust.get())

    def create_button_click(self):
        if self.combo_factor.current() != 0:
            lab = self.combo_labs.get()
            cust = self.combo_cust.get()
            fact = self.combo_factor.get()
            GenerateProtocolWindow(lab, cust, fact)
        else:
            messagebox.showerror('Ошибка', 'Выберите один фактор')

    @staticmethod
    def add_lab_button_click():
        AddLabWindow()

    @staticmethod
    def add_cust_button_click():
        AddCustomerWindow()

    def add_department_button_click(self):
        customer_name = self.tree.heading('customer').get('text')
        AddDepartmentWindow(customer_name)

    def add_workplace_button_click(self):
        if self.tree.focus():
            customer_name = self.tree.heading('customer').get('text')
            if '.' in self.tree.focus():
                department_id = self.tree.parent(self.tree.focus())
            else:
                department_id = self.tree.focus()
            department_name = self.tree.item(department_id).get('values')[0]
            AddWorkplaceWindow(department_id, department_name, customer_name)
        else:
            messagebox.showerror('Ошибка', 'Выберите отдел')
            return 1

    def edit_button_click(self):
        customer_name = self.tree.heading('customer').get('text')
        if '.' in self.tree.focus():
            department_id = self.tree.parent(self.tree.focus())
            department_name = self.tree.item(department_id).get('values')[0]
            wp_name = self.tree.item(self.tree.focus()).get('values')[0]
            wp_id = self.tree.focus()
            wp_id = wp_id.partition('.')[0]
            AddWorkplaceWindow(department_id, department_name, customer_name, wp_name.split('. ')[1], int(wp_id))
        else:
            department_id = self.tree.focus()
            customer_name = self.tree.heading('customer').get('text')
            department_name = self.tree.item(department_id).get('values')[0]
            AddDepartmentWindow(customer_name, department_name, int(department_id))

    def export_button_click(self):
        file = asksaveasfilename(defaultextension=".csv", initialfile="report.csv")
        if file:
            export_workplaces_click(file, self.combo_cust.get(), self.combo_factor.get())
            messagebox.showinfo('Импорт файла', f'Файл {file} успешно сохранен.')

    def import_button_click(self):
        file = askopenfilename()
        if file:
            import_workplaces_click(file, self.combo_factor.get())
            messagebox.showinfo('Импорт файла', f'Файл {file} успешно импортирован.')

    def import_wp_button_click(self):
        file = askopenfilename()
        if file:
            import_wp_click(file, self.combo_cust.get())
            messagebox.showinfo('Импорт файла', f'Файл {file} успешно импортирован.')

    def update_treeview(self, customer_name):
        self.tree.heading('customer', text=customer_name, anchor=CENTER)
        dep_iid = ''
        wa_iid = ''
        count = 1
        if self.combo_factor.get() == '':
            factor = 'Все факторы'
        else:
            factor = self.combo_factor.get()

        customer = get_customer(customer_name, factor)
        for dep in customer.departments:
            dep_iid = str(dep.department_id)
            self.tree.insert(parent='', index='end', iid=dep_iid, values=(dep.name,))
            for wa in dep.working_areas:
                wa_iid = str(wa.number) + '. '
                self.tree.insert(parent=dep_iid, index='end', iid=wa_iid, values=(str(count) + '. ' + wa.name,))
                count += 1

    def generate_tree(self, customer_name):
        if hasattr(self, 'tree'):
            for i in self.tree.get_children():
                self.tree.delete(i)
            self.update_treeview(customer_name)

        else:
            self.grid_frame = tk.Frame(bd=2)
            self.grid_frame.pack(side=tk.TOP, fill=tk.X)

            self.add_department_button = ttk.Button(self.grid_frame, text='Добавить отдел', state='normal',
                                                    command=self.add_department_button_click)
            self.add_department_button.pack(side=tk.LEFT, fill=tk.X, expand=True)

            self.add_workplace_button = ttk.Button(self.grid_frame, text='Добавить РМ', state='normal',
                                                   command=self.add_workplace_button_click)
            self.add_workplace_button.pack(side=tk.LEFT, fill=tk.X, expand=True)

            self.edit_button = ttk.Button(self.grid_frame, text='Изменить', state='normal',
                                          command=self.edit_button_click)
            self.edit_button.pack(side=tk.LEFT, fill=tk.X, expand=True)

            self.tree_frame = tk.Frame(bd=2)
            self.tree_frame.pack(side=tk.TOP, fill=tk.BOTH)

            self.tree = ttk.Treeview(self.tree_frame, height=14, selectmode='browse')
            self.tree['columns'] = ('customer',)
            self.tree.column('#0', width=5)
            self.tree.column('customer', anchor=W, width=635)

            self.update_treeview(customer_name)

            self.tree.pack(side=tk.LEFT, fill=tk.BOTH)

    def init_main_window(self):
        self.toolbar = tk.Frame(bd=2)
        self.toolbar.pack(side=tk.TOP, fill=tk.X)

        lab_list = get_labs_list()

        tk.Label(self.toolbar, text="Лаборатория:").grid(column=0, row=0)
        self.combo_labs = ttk.Combobox(self.toolbar)
        self.combo_labs.configure(values=lab_list)
        self.combo_labs.grid(column=1, row=0)
        self.combo_labs.bind("<<ComboboxSelected>>", self.lab_checked)

        self.add_lab_button = ttk.Button(self.toolbar, text='+', state='normal', width=2,
                                         command=self.add_lab_button_click)
        self.add_lab_button.grid(column=2, row=0)

        tk.Label(self.toolbar, text="Организация:").grid(column=0, row=1)
        self.combo_cust = ttk.Combobox(self.toolbar, state='disabled')
        self.combo_cust.grid(column=1, row=1)
        self.combo_cust.bind("<<ComboboxSelected>>", self.customer_checked)

        self.add_cust_button = ttk.Button(self.toolbar, text='+', state='normal', width=2,
                                          command=self.add_cust_button_click)
        self.add_cust_button.grid(column=2, row=1)

        tk.Label(self.toolbar, text="Производственный фактор:").grid(column=0, row=2)
        self.combo_factor = ttk.Combobox(self.toolbar, state='disabled')
        self.combo_factor.grid(column=1, row=2)
        self.combo_factor.bind("<<ComboboxSelected>>", self.factor_checked)

        self.show_button = ttk.Button(self.toolbar, text='Показать данные', state='disable', width=20,
                                      command=self.show_button_click)
        self.show_button.grid(column=4, row=1)

        self.create_button = ttk.Button(self.toolbar, text='Создать протокол', state='disable', width=20,
                                        command=self.create_button_click)
        self.create_button.grid(column=4, row=2)

        self.import_wp_button = ttk.Button(self.toolbar, text='Импорт РМ', state='disable', width=15,
                                           command=self.import_wp_button_click)
        self.import_wp_button.grid(column=3, row=0)

        self.export_button = ttk.Button(self.toolbar, text='Экспорт зон', state='disable', width=15,
                                        command=self.export_button_click)
        self.export_button.grid(column=3, row=1)

        self.import_button = ttk.Button(self.toolbar, text='Импорт зон', state='disable', width=15,
                                        command=self.import_button_click)
        self.import_button.grid(column=3, row=2)

        tk.Label(self.toolbar, text="").grid(column=0, row=3)

class GenerateProtocolWindow(tk.Toplevel):
    def __init__(self, lab, cust, fact):
        super().__init__()
        self.lab = lab
        self.cust = cust
        self.fact = fact
        self.init_window()

    def add_measuring_button_click(self):
        var = tk.StringVar()
        var.set(get_measure_list(self.fact))
        AddListboxCheckWindow(self.combo_measuring, 'методики', var)

    def add_methodologies_button_click(self):
        var = tk.StringVar()
        var.set(get_methodologies_list(self.fact))
        AddListboxCheckWindow(self.combo_methodologies, 'СИ', var)

    def create_button_click(self):
        zamer = self.combo_zam.get()
        oformitel = self.combo_oformit.get()
        measure = tuple(self.combo_measuring.get().split(' '))
        methodologies = tuple(self.combo_methodologies.get().split(' '))
        date = self.date.get_date()
        date_izm = self.date_zamer.get_date()
        if zamer == '':
            messagebox.showerror('Ошибка', 'Не выбран замерщик.')
            return 1
        elif oformitel == '':
            messagebox.showerror('Ошибка', 'Не выбран оформитель.')
            return 1
        elif measure == '':
            messagebox.showerror('Ошибка', 'Не выбраны приборы.')
            return 1
        elif methodologies == '':
            messagebox.showerror('Ошибка', 'Не выбран методики.')
            return 1
        elif date == '':
            messagebox.showerror('Ошибка', 'Не выбрана дата протокола.')
            return 1
        elif date_izm == '':
            messagebox.showerror('Ошибка', 'Не выбрана дата замера.')
            return 1

        else:
            result = generate_protocol(self.lab, self.cust, self.fact, zamer.split(': ')[0], oformitel.split(':')[0],
                                       measure, methodologies, date, date_izm)
            if result == 0:
                messagebox.showinfo('Создание протокола', 'Протокол успешно создан.')
            elif result == 1:
                messagebox.showerror('Ошибка', 'Не заполнены данные об измерениях и метеообстановке, '
                                               'заполните их и повторите попытку')
            elif result == 2:
                messagebox.showerror('Ошибка', 'Не заполнены данные о метеообстановке, '
                                               'заполните их и повторите попытку')
            elif result == 3:
                messagebox.showerror('Ошибка', 'Не заполнены данные об измерениях, '
                                               'заполните их и повторите попытку')
            self.destroy()

    def init_window(self):
        self.title('Создание протокола')
        self.geometry('420x210+400+300')
        self.resizable(False, False)

        self.grab_set()
        self.focus_set()

        tk.Label(self, text="Выберите замерщика:", bd=5).grid(column=0, row=1)
        self.combo_zam = ttk.Combobox(self, values=get_experts_list(), width=40)
        self.combo_zam.grid(column=1, row=1, columnspan=3)

        tk.Label(self, text="Выберите оформителя:", bd=5).grid(column=0, row=2)
        self.combo_oformit = ttk.Combobox(self, values=get_experts_list(), width=40)
        self.combo_oformit.grid(column=1, row=2, columnspan=3)

        tk.Label(self, text="Выберите приборы:", bd=5).grid(column=0, row=3)

        self.combo_measuring = ttk.Entry(self, width=38)
        self.combo_measuring.grid(column=1, row=3, columnspan=2)

        self.add_measuring_button = ttk.Button(self, text='+', width=3, command=self.add_measuring_button_click)
        self.add_measuring_button.grid(column=3, row=3)

        tk.Label(self, text="Выберите методики:", bd=5).grid(column=0, row=4)

        self.combo_methodologies = ttk.Entry(self, width=38)
        self.combo_methodologies.grid(column=1, row=4, columnspan=2)

        self.add_methodologies_button = ttk.Button(self, text='+', width=3,
                                                   command=self.add_methodologies_button_click)
        self.add_methodologies_button.grid(column=3, row=4)

        tk.Label(self, text="Дата протокола:", bd=5).grid(column=0, row=5)
        self.date = DateEntry(self, width=40, foreground="white", bd=2)
        self.date.grid(column=1, row=5, columnspan=3)

        tk.Label(self, text="Дата замера:", bd=5).grid(column=0, row=6)
        self.date_zamer = DateEntry(self, width=40, foreground="white", bd=2)
        self.date_zamer.grid(column=1, row=6, columnspan=3)

        self.exit_button = ttk.Button(self, text='Отмена', width=15, command=self.destroy)
        self.exit_button.grid(column=1, row=7)

        self.create_button = ttk.Button(self, text='Добавить', width=15, command=self.create_button_click)
        self.create_button.grid(column=2, row=7)

class AddLabWindow(tk.Toplevel):
    def __init__(self):
        super().__init__()
        self.init_window()

    def add_button_click(self):
        if self.entry_short_name.get() == '':
            messagebox.showerror('Ошибка', 'Поле "Сокращенное название" должно быть заполнено.')
            return 1
        elif self.entry_name.get() == '':
            messagebox.showerror('Ошибка', 'Поле "Название полное" должно быть заполнено.')
            return 1
        elif self.entry_name_lab.get() == '':
            messagebox.showerror('Ошибка', 'Поле "Название лаборатории" должно быть заполнено.')
            return 1
        elif self.entry_logo.get() == '':
            messagebox.showerror('Ошибка', 'Поле "Логотип" должно быть заполнено.')
            return 1
        elif self.entry_director.get() == '':
            messagebox.showerror('Ошибка', 'Поле "Руководитель" должно быть заполнено.')
            return 1
        elif self.entry_address.get() == '':
            messagebox.showerror('Ошибка', 'Поле "Адрес" должно быть заполнено.')
            return 1
        elif self.entry_certificate_number.get() == '':
            messagebox.showerror('Ошибка', 'Поле "Номер сертификата" должно быть заполнено.')
            return 1
        elif self.entry_phone.get() == '':
            messagebox.showerror('Ошибка', 'Поле "Телефон" должно быть заполнено.')
            return 1
        elif self.entry_e_mail.get() == '':
            messagebox.showerror('Ошибка', 'Поле "E-mail" должно быть заполнено.')
            return 1
        else:
            add_laboratory_click(self.entry_short_name.get(), self.entry_name.get(), self.entry_name_lab.get(),
                                 self.entry_logo.get(), self.entry_director.get(), self.entry_address.get(),
                                 self.entry_certificate_number.get(), self.entry_phone.get(), self.entry_e_mail.get())
            self.destroy()

    def init_window(self):
        self.title('Добавить лабораторию')
        self.geometry('500x280+400+300')
        self.resizable(False, False)

        self.grab_set()
        self.focus_set()

        tk.Label(self, text="Название сокращенное:", bd=5).grid(column=0, row=1)
        self.entry_short_name = ttk.Entry(self, width=55)
        self.entry_short_name.grid(column=1, row=1, columnspan=2)

        tk.Label(self, text="Название полное:", bd=5).grid(column=0, row=2)
        self.entry_name = ttk.Entry(self, width=55)
        self.entry_name.grid(column=1, row=2, columnspan=2)

        tk.Label(self, text="Название лаборатории:", bd=5).grid(column=0, row=3)
        self.entry_name_lab = ttk.Entry(self, width=55)
        self.entry_name_lab.grid(column=1, row=3, columnspan=2)

        tk.Label(self, text="Логотип:", bd=5).grid(column=0, row=4)
        self.entry_logo = ttk.Entry(self, width=55)
        self.entry_logo.grid(column=1, row=4, columnspan=2)

        tk.Label(self, text="Руководитель:", bd=5).grid(column=0, row=5)
        self.entry_director = ttk.Entry(self, width=55)
        self.entry_director.grid(column=1, row=5, columnspan=2)

        tk.Label(self, text="Адрес:", bd=5).grid(column=0, row=6)
        self.entry_address = ttk.Entry(self, width=55)
        self.entry_address.grid(column=1, row=6, columnspan=2)

        tk.Label(self, text="Номер сертификата:", bd=5).grid(column=0, row=7)
        self.entry_certificate_number = ttk.Entry(self, width=55)
        self.entry_certificate_number.grid(column=1, row=7, columnspan=2)

        tk.Label(self, text="Телефон:", bd=5).grid(column=0, row=8)
        self.entry_phone = ttk.Entry(self, width=55)
        self.entry_phone.grid(column=1, row=8, columnspan=2)

        tk.Label(self, text="E-mail:", bd=5).grid(column=0, row=9)
        self.entry_e_mail = ttk.Entry(self, width=55)
        self.entry_e_mail.grid(column=1, row=9, columnspan=2)

        self.exit_button = ttk.Button(self, text='Отмена', width=20, command=self.destroy)
        self.exit_button.grid(column=1, row=10)

        self.create_button = ttk.Button(self, text='Создать', width=20, command=self.add_button_click)
        self.create_button.grid(column=2, row=10)

class AddCustomerWindow(tk.Toplevel):
    def __init__(self):
        super().__init__()
        self.init_window()

    def add_button_click(self):
        if self.entry_short_name.get() == '':
            messagebox.showerror('Ошибка', 'Поле "Сокращенное название" должно быть заполнено.')
            return 1
        elif self.entry_name.get() == '':
            messagebox.showerror('Ошибка', 'Поле "Название полное" должно быть заполнено.')
            return 1
        elif self.entry_legal_address.get() == '':
            messagebox.showerror('Ошибка', 'Поле "Юридический адрес" должно быть заполнено.')
            return 1
        elif self.entry_actual_address.get() == '':
            messagebox.showerror('Ошибка', 'Поле "Фактический адрес" должно быть заполнено.')
            return 1
        elif self.entry_contract_number.get() == '':
            messagebox.showerror('Ошибка', 'Поле "Номер договора" должно быть заполнено.')
            return 1
        elif self.entry_contract_date.get() == '':
            messagebox.showerror('Ошибка', 'Поле "Дата договора" должно быть заполнено.')
            return 1
        else:
            add_customer_click(self.entry_short_name.get(), self.entry_name.get(), self.entry_legal_address.get(),
                               self.entry_actual_address.get(), self.entry_contract_number.get(),
                               self.entry_contract_date.get())
            self.destroy()

    def init_window(self):
        self.title('Добавить заказчика')
        self.geometry('500x220+400+300')
        self.resizable(False, False)

        self.grab_set()
        self.focus_set()

        tk.Label(self, text="Название сокращенное:", bd=5).grid(column=0, row=1)
        self.entry_short_name = ttk.Entry(self, width=55)
        self.entry_short_name.grid(column=1, row=1, columnspan=2)

        tk.Label(self, text="Название полное:", bd=5).grid(column=0, row=2)
        self.entry_name = ttk.Entry(self, width=55)
        self.entry_name.grid(column=1, row=2, columnspan=2)

        tk.Label(self, text="Юридический адрес:", bd=5).grid(column=0, row=3)
        self.entry_legal_address = ttk.Entry(self, width=55)
        self.entry_legal_address.grid(column=1, row=3, columnspan=2)

        tk.Label(self, text="Фактический адрес:", bd=5).grid(column=0, row=4)
        self.entry_actual_address = ttk.Entry(self, width=55)
        self.entry_actual_address.grid(column=1, row=4, columnspan=2)

        tk.Label(self, text="Номер договора:", bd=5).grid(column=0, row=5)
        self.entry_contract_number = ttk.Entry(self, width=55)
        self.entry_contract_number.grid(column=1, row=5, columnspan=2)

        tk.Label(self, text="Дата договора:", bd=5).grid(column=0, row=6)
        self.entry_contract_date = DateEntry(self, width=50, foreground="white", bd=2)
        self.entry_contract_date.grid(column=1, row=6, columnspan=2)

        self.exit_button = ttk.Button(self, text='Отмена', width=20, command=self.destroy)
        self.exit_button.grid(column=1, row=7)

        self.create_button = ttk.Button(self, text='Создать', width=20, command=self.add_button_click)
        self.create_button.grid(column=2, row=7)

class AddDepartmentWindow(tk.Toplevel):
    def __init__(self, customer_name, department_name='', department_id=0):
        super().__init__()
        self.customer_name = customer_name
        self.department_name = department_name
        self.department_id = department_id
        self.init_window()

    def add_button_click(self):
        if self.entry_department_name.get() == '':
            messagebox.showerror('Ошибка', 'Поле "Название отдела" должно быть заполнено.')
            return 1
        else:
            if self.department_id == 0:
                add_department_click(self.entry_department_name.get(), self.customer_name)
            else:
                edit_department_click(self.department_id, self.entry_department_name.get())
            self.destroy()

    def init_window(self):
        self.title('Добавить отдел')
        self.geometry('500x130+400+300')
        self.resizable(False, False)

        self.grab_set()
        self.focus_set()

        tk.Label(self, text=self.customer_name, bd=5).grid(column=0, row=0)

        tk.Label(self, text="Название отдела:", bd=5).grid(column=0, row=1)
        self.entry_department_name = ttk.Entry(self, width=55)
        self.entry_department_name.insert(0, self.department_name)
        self.entry_department_name.grid(column=1, row=1, columnspan=2)

        self.exit_button = ttk.Button(self, text='Отмена', width=20, command=self.destroy)
        self.exit_button.grid(column=1, row=2)

        self.create_button = ttk.Button(self, text='Создать', width=20, command=self.add_button_click)
        self.create_button.grid(column=2, row=2)

class AddWorkplaceWindow(tk.Toplevel):
    def __init__(self, department_id, department_name, customer_name, wp_name='', wp_id=0):
        super().__init__()
        self.department_id = department_id
        self.customer_name = customer_name
        self.department_name = department_name
        self.wp_name = wp_name
        self.wp_id = wp_id
        self.init_window()

    def add_button_click(self):
        if self.entry_wp_name.get() == '':
            messagebox.showerror('Ошибка', 'Поле "Название точки" должно быть заполнено.')
            return 1
        elif not self.check_noise.get() and not self.check_loc_vib.get() and not self.check_gen_vib.get():
            messagebox.showerror('Ошибка', 'Необходимо выбрать как минимум один фактор.')
            return 1
        else:
            if self.wp_id != 0:
                edit_workplace_click(self.wp_id, self.entry_wp_name.get(), self.check_noise.get(),
                                     self.check_loc_vib.get(), self.check_gen_vib.get())
            else:
                add_workplace_click(self.entry_wp_name.get(), self.department_id, self.check_noise.get(),
                                    self.check_loc_vib.get(), self.check_gen_vib.get())
            self.destroy()

    def init_window(self):
        self.title('Добавить рабочую зону')
        self.geometry('400x200+400+300')
        self.resizable(False, False)

        self.grab_set()
        self.focus_set()

        tk.Label(self, text=self.customer_name + ' - ' + self.department_name, bd=5). \
            grid(column=0, row=0, columnspan=3, sticky=W)

        tk.Label(self, text="Название точки:", bd=5).grid(column=0, row=2, sticky=W)
        self.entry_wp_name = ttk.Entry(self, width=40)
        self.entry_wp_name.insert(0, self.wp_name)
        self.entry_wp_name.grid(column=1, row=2, columnspan=2)

        tk.Label(self, text="Выберите факторы:", bd=5).grid(column=0, row=3, sticky=W)
        self.check_noise = tk.BooleanVar()
        self.check_noise.set(0)
        self.check_loc_vib = tk.BooleanVar()
        self.check_loc_vib.set(0)
        self.check_gen_vib = tk.BooleanVar()
        self.check_gen_vib.set(0)
        ttk.Checkbutton(self, text='Шум', variable=self.check_noise, onvalue=1, offvalue=0). \
            grid(column=1, row=3, columnspan=2, sticky=W)
        ttk.Checkbutton(self, text='Вибрация локальная', variable=self.check_loc_vib, onvalue=1, offvalue=0). \
            grid(column=1, row=4, columnspan=2, sticky=W)
        ttk.Checkbutton(self, text='Вибрация общая', variable=self.check_gen_vib, onvalue=1, offvalue=0). \
            grid(column=1, row=5, columnspan=2, sticky=W)

        tk.Label(self, text=" ", bd=5).grid(column=0, row=6, columnspan=3)

        self.exit_button = ttk.Button(self, text='Отмена', width=20, command=self.destroy)
        self.exit_button.grid(column=1, row=6)

        self.create_button = ttk.Button(self, text='Создать', width=20, command=self.add_button_click)
        self.create_button.grid(column=2, row=6)

class AddListboxCheckWindow(tk.Toplevel):
    def __init__(self, parent_window_box, name, var):
        super().__init__()
        self.name = name
        self.var = var
        self.result = []
        self.parent_window_box = parent_window_box
        self.init_window()

    def add_button_click(self):
        if not self.list_box.curselection():
            messagebox.showerror('Ошибка', 'Выберите хотя бы один пункт.')
            return 1
        else:
            items_list = self.list_box.curselection()
            for item in items_list:
                self.result.append(self.list_box.get(item).split(': ')[0])
            self.parent_window_box.insert(0, self.result)
            self.destroy()

    def init_window(self):
        self.title(f'Выберите {self.name}')
        self.geometry('370x190+400+300')
        self.resizable(False, False)

        self.grab_set()
        self.focus_set()

        tk.Label(self, text=f'Выберите {self.name}', bd=5).grid(column=0, row=0)
        self.list_box = tk.Listbox(self, listvariable=self.var, width=60, height=8, selectmode='extended')
        self.list_box.grid(column=0, row=1, columnspan=2)

        self.exit_button = ttk.Button(self, text='Отмена', width=20, command=self.destroy)
        self.exit_button.grid(column=0, row=2)

        self.create_button = ttk.Button(self, text='Добавить', width=20, command=self.add_button_click)
        self.create_button.grid(column=1, row=2)

    def __del__(self):
        return self.result


from bs4 import BeautifulSoup as soup
from crawler import crawl
from utils import strip_domain
import string
import urllib2
import re

# Finds any emails in domain source code
def parse_HTML(html):
	emails = list()
	s = soup(html, 'html.parser')
	for s_elem in s.strings:
		for word in s_elem.split():
			match = re.match(r"(^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$)", word)
			if match:
				emails.append(match.group(0).encode('utf-8').strip())

	# Return emails
	return [email.string.encode('utf-8').strip() for email in s.select('a[href^=mailto]')] + emails

# Generates usernames with a higher weightage for more probable usernames
def username_generator(first_name, last_name, middle_name=None, domains=[], linkedin_url=None, angellist_url=None, twitter_url=None, facbeook_url=None, github_url=None):
	usernames = dict()
	username_chars = set([char for char in string.ascii_lowercase] + [str(i) for i in range(10)] + ['_', '-', '.'])

	# Adds usernames to the username set
	def add_username(username, link=False):
		username = username.lower().replace('-', '_')
		if link:
			if username in usernames:
				usernames[username] += 3
			else:
				usernames[username] = 3
		if username in usernames:
			usernames[username] += 1
		else:
			usernames[username] = 1

	# Extracts usernames from URLs
	def extract_username(url, stub):
		url = url.lower()
		if stub in url:
			start = url.index(stub) + len(stub)
			end = start
			while end < len(url) and url[end] in username_chars:
				end += 1

			username = url[start:end]
			add_username(username, link=True)

	for url in [(linkedin_url, 'linkedin.com/in/'), (angellist_url, 'angel.co/'), (twitter_url, 'twitter.com/'), (github_url, 'github.com/'), (facbeook_url, 'facebook.com/')]:
		if url[0]:
			extract_username(url[0], url[1])

	# Common usernames
	add_username(first_name + last_name)
	add_username(first_name[0] + last_name)
	add_username(first_name + last_name[0])
	add_username(first_name + '_' + last_name)
	add_username(first_name[0] + '_' + last_name)
	add_username(first_name + '_' + last_name[0])
	add_username(first_name + '.' + last_name)
	add_username(first_name[0] + '.' + last_name)
	add_username(first_name + '.' + last_name[0])

	if middle_name:
		add_username(first_name + middle_name + last_name)
		add_username(first_name + middle_name[0] + last_name)
		add_username(first_name[0] + middle_name + last_name)
		add_username(first_name[0] + middle_name[0] + last_name)
		add_username(first_name + middle_name + last_name[0])
		add_username(first_name + middle_name[0] + last_name[0])
		add_username(first_name + '_' + middle_name + '_' + last_name)
		add_username(first_name + '_' + middle_name[0] + '_' + last_name)
		add_username(first_name[0] + '_' + middle_name + '_' + last_name)
		add_username(first_name[0] + '_' + middle_name[0] + '_' + last_name)
		add_username(first_name + '.' + middle_name + '.' + last_name)
		add_username(first_name + '.' + middle_name[0] + '.' + last_name)
		add_username(first_name[0] + '.' + middle_name + '.' + last_name)
		add_username(first_name[0] + '.' + middle_name[0] + '.' + last_name)

	for domain in domains:
		if domain[:4] != 'http':
			domain = 'http://' + domain
		stripped_domain = strip_domain(domain)
		add_username(stripped_domain[:stripped_domain.index('.')])

	# Weights usernames on their content
	def username_weight(username):
		letters = False
		numbers = False
		symbols = False
		num_strs = map(str, range(10))
		chars = set(username)
		for char in chars:
			if char in string.ascii_lowercase:
				letters = True
			elif char in num_strs:
				numbers = True
			else:
				symbols = True
		if letters:
			if numbers:
				if symbols:
					return 0.7 # letters, numbers, and symbols
				return 0.85 # letters and numbers
			elif symbols:
				return 0.85 # letters and symbols
			return 1.0 # letters only
		elif numbers:
			if symbols:
				return 0.2 # numbers and symbols
			return 0.3 # numbers only
		else:
			return 0.1 # symbols only

	for username in usernames:
		usernames[username] *= username_weight(username)

	return usernames

# Generates potential email addresses
def email_generator(usernames, domains=[], links=[]):
	emails = dict()
	max_confidence = max(usernames.values())
	common_domains = ['gmail.com', 'yahoo.com', 'hotmail.com', 'aol.com']

	username_from_email = lambda email: email[:email.index('@')]

	for domain in domains:
		if domain[:4] != 'http':
			domain = 'http://' + domain
		# Might not have permission to scrape
		try:
			internal_links = crawl(domain)
			for l in internal_links:
				resp = urllib2.urlopen(l).read()
				parsed_emails = parse_HTML(resp)
				for email in parsed_emails:
					emails[email] = max_confidence
		except:
			pass

		# Common emails of people with their own domains
		# If any of these exist, the chance of it being their email
		# is highly likely since it's a personal domain
		stripped_domain = strip_domain(domain)
		emails['admin@' + stripped_domain] = max_confidence
		emails['info@' + stripped_domain] = max_confidence
		emails['me@' + stripped_domain] = max_confidence

	for link in links:
		if link:
			if link[:4] != 'http':
				link = 'http://' + link
			# Might not have permission to scrape
			try:
				resp = urllib2.urlopen(link).read()
				parsed_emails = parse_HTML(resp)
				for email in set(parsed_emails):
					if email in emails:
						emails[email] *= 1.5
					else:
						emails[email] = max_confidence
			except:
				pass

	for username in usernames:
		for domain in common_domains:
			email = username + '@' + domain
			if email in emails:
				emails[username + '@' + domain] += usernames[username]
			else:
				emails[username + '@' + domain] = usernames[username]

	return emails


#! /usr/bin/env python
# -*- coding: utf-8 -*-

import csv
import os
import cv2
import numpy as np
import math

import mylogger

def plotShotPoses(shots, floorplan, dataset_path, out_fn, ref_shots={}):
    """
    :type shots: dict[str, scripts.myopensfm.types.Shot]
    :type floorplan: scripts.myopensfm.types.Floorplan
    :type dataset_path: str
    :type out_fn: str
    :type ref_shots: dict[str, scripts.myopensfm.types.Shot]
    """

    floorplans_path = os.path.join(dataset_path, "floorplans")

    floorplan_img = cv2.imread(os.path.join(floorplans_path, floorplan.id))
    # floorplan_img *= 0
    # floorplan_img += 255

    for shot in ref_shots:
        pose = ref_shots[shot].pose
        # load and convert to image coordinate
        x, y = floorplan.pose2pix(pose)
        cv2.circle(floorplan_img, (x, y),  5, (0, 0, 0), -1)

    for shot in shots:
        pose = shots[shot].pose
        # load and convert to image coordinate
        x, y = floorplan.pose2pix(pose)
        cv2.circle(floorplan_img, (x, y),  5, (255, 0, 0), -1)

    cv2.imwrite(os.path.join(dataset_path, out_fn), floorplan_img)
    mylogger.logger.info("save " + os.path.join(dataset_path, out_fn))

def save2DTrajectory(shots, floorplan, dataset_path, out_fn):
    """
    :type shots: dict[str, scripts.myopensfm.types.Shot]
    :type floorplan: scripts.myopensfm.types.Floorplan
    :type dataset_path: str
    :type out_fn: str
    """

    floorplans_path = os.path.join(dataset_path, "floorplans")
    floorplan_img = cv2.imread(os.path.join(floorplans_path, floorplan.id))

    datas = []

    for shot in shots:
        pose = shots[shot].pose
        # load and convert to image coordinate
        x, y = floorplan.pose2pix(pose)
        z = floorplan.heightInMeter(pose)

        r_vec = shots[shot].viewing_direction()
        angle_rad = np.arctan2(r_vec[0], r_vec[1])

        datas.append((shot, x, y, shots[shot].metadata.capture_time, z, angle_rad))
    datas.sort()

    with open(os.path.join(dataset_path, out_fn), 'w') as f:
        writer = csv.writer(f, lineterminator='\n')
        writer.writerows(datas)
        mylogger.logger.info("save " + os.path.join(dataset_path, out_fn))


#!/usr/bin/env python3

import argparse
import glob
import itertools
import json
import multiprocessing
import multiprocessing.pool
import os
import pprint
import re
import shutil
import subprocess
import sys

clang_tidy_possible_names = ["clang-tidy-7.0",
                             "clang-tidy-6.0",
                             "clang-tidy-5.0",
                             "clang-tidy-4.0",
                             "clang-tidy-3.9",
                             "clang-tidy-3.8",
                             "clang-tidy"]

clang_tidy_exec = ""
clang_tidy_checks = ["boost*",
                     "bugprone*",
                     "cert*",
                     "clang-analizer-*",
                     "clang-diagnostic-*",
                     "cppcoreguidelines*",
                     "-cppcoreguidelines-pro-bounds-constant-array-index",
                     "google*",
                     "llvm*",
                     "-llvm-header-guard",
                     "misc*",
                     "modernize*",
                     "performance*",
                     "-performance-move-const-arg",
                     "readability*"]

folders_to_run = ["."]
extentions_to_run = [".cpp", ".cxx", ".cc"]

warning_regex = "([^\\s]+):(\\d+):(\\d+): (warning|note|error|fatal error): ((?:operator\\[\\]|[^\\n\\[])+)(?:\\[([^\\]]+)\\])?"
warning_regex_program = re.compile(warning_regex, re.ASCII | re.MULTILINE)

clang_tidy_build_dir = None

clang_tidy_raw_options = []

verbose = False
raw = False

def find_clang_tidy():
    global clang_tidy_exec

    for name in clang_tidy_possible_names:
        path = shutil.which(name)
        if (path is not None):
            clang_tidy_exec = path
            break

    if (len(clang_tidy_exec) == 0):
        print("Clang Tidy not found. Please install to path.", file=sys.stderr)
        exit(1)

    if (verbose):
        print("Clang Tidy found at: {}\n".format(clang_tidy_exec), file=sys.stderr)

def read_settings(filename):
    global clang_tidy_checks
    global folders_to_run

    try:
        with open(filename, 'r') as f:
            if (verbose):
                print("Settings file: {} found.".format(filename), file=sys.stderr)

            settings = json.load(f)

            if ("checks" in settings):
                clang_tidy_checks = settings["checks"]
            if ("folders" in settings):
                folders_to_run = settings["folders"]
    except FileNotFoundError:
        if (verbose):
            print("Settings file: {} not found.".format(filename), file=sys.stderr)

def find_all_files(input_files):
    if (input_files is not None):
        return [os.path.normpath(infile) for infile in input_files]

    file_list = []

    if (verbose):
        print("File glob patterns: ", file=sys.stderr)

    for folder in folders_to_run:
        for ext in extentions_to_run:
            glob_str = folder + "/**/*" + ext
            if (verbose):
                print(glob_str, file=sys.stderr)
            file_list.extend(glob.glob(glob_str, recursive=True))
    return file_list

class ClangTidyError(Exception):
    pass

def run_file(filename):
    full_name = os.path.join(os.getcwd(), filename)

    folders = "({}).*".format("|".join([os.path.abspath(folder)
                                        for folder in folders_to_run]))

    checks = "-checks={}".format(",".join(clang_tidy_checks))

    if (clang_tidy_build_dir is not None):
        build_arg = "-p={}".format(clang_tidy_build_dir)
    else:
        build_arg = ""

    args = [clang_tidy_exec, checks, "-header-filter={}".format(folders), full_name, build_arg] + clang_tidy_raw_options

    if (verbose):
        print(args, file=sys.stderr)
    else:
        print("{} {}".format(clang_tidy_exec, filename), file=sys.stderr)

    clang_tidy_result = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

    if (raw or clang_tidy_result.returncode != 0):
        sys.stdout.write(str(clang_tidy_result.stdout, 'utf-8'))
        sys.stderr.write(str(clang_tidy_result.stderr, 'utf-8'))

    if (clang_tidy_result.returncode != 0):
        raise ClangTidyError("{} returned non-zero.".format(clang_tidy_exec))

    return str(clang_tidy_result.stdout, 'utf-8')

class ParsedOutput:
    diagnostic_text = ""
    file = ""
    line = 0
    char = 0
    diagnostic_type = ""
    error = ""
    check = ""
    begin_match = 0
    end_match = 0
    full_text = ""

def parse_output(raw_output):
    match_iter = warning_regex_program.finditer(raw_output)

    matches = []

    for match in match_iter:
        po = ParsedOutput()

        po.diagnostic_text = match.group(0)
        po.file = match.group(1)
        po.line = int(match.group(2))
        po.char = int(match.group(3))
        po.diagnostic_type = match.group(4)
        po.error = match.group(5)
        po.check = match.group(6)
        po.begin_match = match.start()
        po.end_match = match.end()

        matches.append(po)

    if len(matches) > 0:
        for elem1, elem2 in zip(matches, matches[1:]):
            #  Plus one to remove the newline
            elem1.full_text = raw_output[elem1.end_match + 1:elem2.begin_match - 1]

        matches[-1].full_text = raw_output[matches[-1].end_match + 1:-1]

    combined_matches = []

    for elem in matches:
        if elem.diagnostic_type == "note":
            try:
                combined_matches[-1].append(elem)
            except IndexError as e:
                print(combined_matches)
                print(elem.full_text)
                raise e
        else:
            combined_matches.append([elem])

    return combined_matches

def recolor_output(parsed_output):
    if (parsed_output.diagnostic_type == "note"):
        warning_color = "\033[30;1m"
    elif (parsed_output.diagnostic_type == "warning"):
        warning_color = "\033[35;1m"
    else:
        warning_color = "\033[31;1m"

    if (parsed_output.check is not None):
        check_string = "[{}]".format(parsed_output.check)
    else:
        check_string = ""

    header = "\033[0;1m{}:{}:{}: {}{}: \033[0;1m{} {}\033[0m".format(parsed_output.file,
                                                                     parsed_output.line,
                                                                     parsed_output.char,
                                                                     warning_color,
                                                                     parsed_output.diagnostic_type,
                                                                     parsed_output.error,
                                                                     check_string)

    if (len(parsed_output.full_text) > 0):
        # first line is normal text
        # second line bold green
        # third line is normal green
        # no newline at beginning of fulltext
        body = re.sub("^(.*)\n(.*)(\n)?",
                      "\\1\n\033[0;32;1m\\2\\3\033[0;32m",
                      parsed_output.full_text,
                      1) \
               + "\033[0m"

        return "{}\n{}".format(header, body)
    else:
        return header

def nocolor_output(parsed_output):
    if (parsed_output.check is not None):
        check_string = "[{}]".format(parsed_output.check)
    else:
        check_string = ""

    header = "{}:{}:{}: {}: {} {}".format(parsed_output.file,
                                          parsed_output.line,
                                          parsed_output.char,
                                          parsed_output.diagnostic_type,
                                          parsed_output.error,
                                          check_string)

    if (len(parsed_output.full_text) > 0):
        return "{}\n{}".format(header, parsed_output.full_text)
    else:
        return header

def main():
    global clang_tidy_exec
    global clang_tidy_raw_options
    global clang_tidy_build_dir
    global folders_to_run
    global raw
    global verbose

    parser = argparse.ArgumentParser(description="Allows easier use of clang-tidy through multithreading and"
                                                 "elimination of duplicates from output.",
                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)

    parser.add_argument("-j",
                        metavar="jobs",
                        type=int,
                        nargs='?',
                        default=(multiprocessing.cpu_count() + 1),
                        help="Amount of jobs to run")
    parser.add_argument("--folders",
                        metavar="folder",
                        type=str,
                        nargs='+',
                        help="Folders to run through clang-tidy, recursively")
    parser.add_argument("--files",
                        metavar="file.cpp",
                        type=str,
                        nargs='*',
                        help="Files to lint")
    parser.add_argument("-f", "--fix",
                        action="store_true",
                        default=False,
                        help="Automatically fix problems.")
    parser.add_argument("-s", "--format-style",
                        metavar="style",
                        type=str,
                        nargs='?',
                        default="none",
                        help="clang-format style to format code in. Possible options: none/file/{ <json> },llvm,etc.")
    parser.add_argument("-p", "-bd", "--build_dir",
                        metavar="/path/to/build/tree",
                        type=str,
                        nargs='?',
                        help="Folder to find compile-commands.json in. Defaults to $PWD")
    parser.add_argument("-r", "--raw",
                        action="store_true",
                        help="Dump raw output for clang-tidy to stdout and stderr")
    parser.add_argument("-v", "--verbose",
                        action="store_true",
                        help="Print extra information")
    parser.add_argument("-nc", "--no-color",
                        action="store_true",
                        help="Don't recolor output")
    parser.add_argument("--path-to-clang-tidy",
                        metavar="path",
                        type=str,
                        help="Path to clang-tidy executable")
    parser.add_argument("--config",
                        metavar=".clang-tidy-manager",
                        default=".clang-tidy-manager",
                        type=str,
                        help="Path to configuration file")
    parser.add_argument("--raw-options",
                        nargs=argparse.REMAINDER,
                        help="Pass remaining arguments right to clang-tidy")

    args = parser.parse_args()

    if (args.raw_options):
        clang_tidy_raw_options = args.raw_options

    if (args.build_dir is not None):
        clang_tidy_build_dir = args.build_dir

    raw = args.raw
    verbose = args.verbose

    if (args.path_to_clang_tidy is not None):
        clang_tidy_exec = args.path_to_clang_tidy
    else:
        find_clang_tidy()

    read_settings(args.config)

    if (args.folders is not None):
        folders_to_run = args.folders

    jobs = args.j

    if (args.fix):
        clang_tidy_raw_options.append("--fix")
        if (jobs > 1):
            print("Multiple jobs not recommended when fixing problems. Dropping to single job.", file=sys.stderr)
            jobs = 1

    clang_tidy_raw_options.append("-format-style={}".format(args.format_style))

    tp = multiprocessing.pool.ThreadPool(jobs)

    raw_outputs = tp.map(run_file, find_all_files(args.files))
    parsed_outputs = map(parse_output, raw_outputs)

    #  flatten into one error list
    flattened_parsed = sorted(itertools.chain.from_iterable(parsed_outputs),
                              key=lambda warning: (warning[0].file, warning[0].line, warning[0].char, warning[0].error))

    unique_dictionary = {warning[0].diagnostic_text: warning for warning in flattened_parsed}
    unique_list = list(unique_dictionary.values())

    flattened_unique = list(itertools.chain.from_iterable(unique_list))

    if (not args.no_color):
        output = list(map(recolor_output, flattened_unique))
    else:
        output = list(map(nocolor_output, flattened_unique))

    for warning in output:
        print(warning)

    print("\n{} warnings/errors.".format(len(unique_list)))
    print("{} notes.".format(len(flattened_unique) - len(unique_list)))

if __name__ == "__main__":
    main()


import logging
from os import environ

log = logging.getLogger(__name__)

def load_env_vars(config):
    try:
        config["SQLALCHEMY_DATABASE_URI"] = environ.get("DB_URL")
        config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
        config["SQLALCHEMY_ECHO"] = False
        config["PROPAGATE_EXCEPTIONS"] = True
        config["TMP_LOC"] = environ.get("TMP_LOC", "/tmp")
        config['AUTH_PUBLIC_KEY'] = environ.get('AUTH_PUBLIC_KEY')
        config["SERVER_PORT"] = environ.get("SERVER_PORT", 9095)
        config["USE_SYSLOG"] = environ.get("USE_SYSLOG", "True")

        return config
    except Exception as e:
        log.debug(f"error while loading env variables {e}")


import json
import re
import subprocess
import time
from pathlib import Path

import fasttext
import pickledb
from gensim.models import FastText

from src.baselines import utils

# implementation guided by https://github.com/lbozarth/keywordexpansion/blob/main/src_pipelines/run_cosine_similarity.py
TRAIN_FILEPATH = ".tmp.train_w2v.txt"
MODEL_FILEPATH = ".tmp.model_w2v"

def setup_model(tweets):
    if len(tweets) == 0:
        raise RuntimeError("No tweets provided")

    # write tweets to file for processing
    with open(TRAIN_FILEPATH, "w") as f:
        f.write("\n".join(tweets))

    # preprocess model using command line, output to same file
    subprocess.run(
        [
            "cat",
            TRAIN_FILEPATH,
            "|",
            "sed",
            "-e",
            "s/\([.\!?,'/()]\)/ \1 /g",
            "|",
            "tr",
            "[:upper:]",
            "[:lower:]",
            ">",
            TRAIN_FILEPATH,
        ],
        capture_output=True,
    )
    w2v_model = fasttext.train_unsupervised(
        TRAIN_FILEPATH, model="skipgram", minCount=1, dim=300, loss="hs"
    )
    w2v_model.save_model(MODEL_FILEPATH)

def process_tweets(tweets):
    # store as list of lists of words
    processed_tweets = []
    for tweet in tweets:
        tweet = re.sub(r"\([^)]*\)", "", tweet)
        tokens = re.sub(r"[^a-z0-9]+", " ", tweet.lower()).split()
        processed_tweets.append(tokens)

    return processed_tweets

def get_top_keywords(
    tweets, budget, positive_labelled_keywords, negative_labelled_keywords
):
    if len(tweets) <= 10:
        return {}

    # https://towardsdatascience.com/word-embedding-with-word2vec-and-fasttext-a209c1d3e12c

    # setup_model(tweets)
    # fasttext_model = load_facebook_model(MODEL_FILEPATH)

    tweets = process_tweets(tweets)
    fasttext_model = FastText(
        tweets, vector_size=300, min_count=1, workers=4, hs=1, sg=1
    )

    top_keywords = {}
    iter_count = 0
    # retrieve list of most similar to positive & dissimilar from negative keywords
    while len(top_keywords) < budget and iter_count < 4:
        top_keywords.update(
            {
                r[0]: round(r[1], 3)
                for r in fasttext_model.wv.most_similar(
                    positive=list(positive_labelled_keywords),
                    # negative=list(negative_labelled_keywords),
                    topn=budget * (iter_count + 1),
                )
                if r[0] not in positive_labelled_keywords
                and r[0] not in negative_labelled_keywords
            }
        )

        # if need to increase the number of keywords to be included
        iter_count += 1

    # remove keywords that have already been retrieved
    return utils.kget(top_keywords, budget)

if __name__ == "__main__":
    # looking at first day of data for testing purposes
    db_filepath = Path(".tmp.db/tweets_1.db")
    tweet_db = pickledb.load(db_filepath, False)
    tweets = [tweet_db.get(key) for key in tweet_db.getall()]
    print(f"Loaded {len(tweets)} tweets for test")

    start = time.time()

    with open("keywords.json", "r") as f:
        keywords = json.load(f)

    top_keywords = get_top_keywords(tweets, 30, keywords.get("LOCKDOWN"), set())
    print(
        f"Found {len(top_keywords)} top keywords to use in {time.time()-start:.2f} seconds"
    )
    print(json.dumps(top_keywords, indent=4))


import requests
from bs4 import BeautifulSoup as BS
import codecs
import time

# session = requests.Session()
# headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 5.1; rv:47.0) Gecko/20100101 Firefox/47.0','Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'}
url = 'http://top.bigmir.net/report/16936420/'
# resp = session.get(url, headers=headers)

data = ''
handle = codecs.open('bigmir_.html','r', 'utf-8')
data = handle.read()
handle.close()
stat = {}
stat['bigmir'] = {}
stat['bigmir']['xls_url'] = url + '?ver=&format=xls'
if True: # resp.status_code == 200:
    # bsObj = BS(resp.content, "html.parser")
    bsObj = BS(data, "html.parser")
    tables = bsObj.find_all('table', attrs={'class':'small'})
    general_stat = tables[0]
    detail_stat = tables[1]
    raw_data = general_stat.find_all('tr')
    general = []
    for tr in raw_data:
        value = tr.find('td', attrs={'class':'text_right'}).text.replace('\n', '').strip()
        title = tr.find('td', attrs={'class':'large'}).text.replace('\n', '').strip()
        general.append({'title': title, 'value': value})
    stat['bigmir']['general'] = general
    detail = []
    raw_data = detail_stat.find_all('tr')
    size = len(raw_data)
    for i in range(1, size):
        tr = raw_data[i]
        v = tr.find_all('td', attrs={'class': False})
        # print(v)
        title = tr.find('td', attrs={'class':'large'}).text.replace('\n', '').strip()
        values = {}
        values['today'] = v[0].text.replace('\n', '').strip()
        if i < size - 1:
            values['1_day_ago'] = v[1].text.replace('\n', '').strip()
            values['2_days_ago'] = v[2].text.replace('\n', '').strip()
            values['7_days_ago'] = v[3].text.replace('\n', '').strip()
            values['30_days_ago'] = v[4].text.replace('\n', '').strip()
        detail.append({'title': title, 'values': values})
    stat['bigmir']['detail'] = detail

# data = bsObj.prettify()#.encode('utf8')
handle = codecs.open('bigmir_stat.txt', "w", 'utf-8')
handle.write(str(stat))
handle.close()
# data = str(groups)
# handle = codecs.open('bigmir_groups.txt', "w", 'utf-8')
# handle.write(str(data))
# handle.close()


# -*- coding: utf-8 -*-

import sys
from pymongo import MongoClient

def get_rank(user_id):
    client = MongoClient()
    db = client.shiyanlou
    contests = db.contests
    global all_con
    for a in db.contests.find():
        if all_con.get(a["user_id"]) is None:
            temp=[a["user_id"],a["challenge_id"],a["score"],a["submit_time"],0]
            all_con[a["user_id"]]=temp
        else:
            temp=all_con.get(a["user_id"])
            temp[2]=temp[2]+a["score"]
            temp[3]=temp[3]+a["submit_time"]
    all_user=[]
    for value in all_con.values():
        all_user.append(value)
    all_user1=sorted(all_user,key=lambda a:(a[2],-a[3]),reverse=True)
    for t in all_user1:
        t[4]=all_user1.index(t)+1
    for tt in all_user1:
        all_con.get(tt[0])[4]=tt[4]
    need=all_con.get(user_id)
    if not need:
        print("Parameter Error")
        sys.exit()
    rank=need[4]
    score=need[2]
    submit_time=need[3]
    return rank, score, submit_time

if __name__ == "__main__":
    all_con={}
    user_id=0
    if len(sys.argv)!=2:
        print("Parameter Error")
        sys.exit()
    else:
        if not sys.argv[1].isdigit():
            print("Parameter Error")
            sys.exit()
        user_id=int(sys.argv[1])
    userdata = get_rank(user_id)
    print(userdata)


# coding: utf-8

from json import load
from os import path

import logging
log = logging.getLogger('justine')

def cargar_credenciales():
    token = 'eyJyb2wiOiAiYWRtaW5pc3RyYWRvciIsICJkaXJlY2Npb24iOiAiYWxvcnRpeiJ9.3wR/qMGedccms7xFXN+GCbxlhbTknXGaBrtK3byOzJ0='
    cabecera = 'WWW-Authorization'
    return {cabecera: token}

def cargar_datos(ente):
    directorio = path.dirname(__file__)
    ruta = directorio.split('tests')[0] + 'tests/'

    archivo = ruta + 'datos.json'
    fichero = open(archivo, 'rb')
    contenido = load(fichero)

    return contenido[ente]


import sqlite3
from scrolling import clear

def artistsMainMenu(aid, conn):
    # allows the artist to add songs and find top fans/playlists

    # display options
    print("------Artist Main Menu-------")
    action = input("Enter\n1 add a song\n2 find top fans and playlists\n3 to logout\nAnything else to exit the program\n")

    # do the corresponding action
    if action == '1':
        clear()
        addSong(aid, conn)
        return artistsMainMenu(aid, conn)
    elif action == '2':
        clear()
        displayUsersPlaylists(aid, conn)
        return artistsMainMenu(aid, conn)
    elif action == '3':
        clear()
        print("logged out")
        return '4' # should return to main screen
    else:
        return -1

def displayUsersPlaylists(aid, conn):
    # display top fans and playlists

    # display top three users
    print("------Top Three Users------")
    users = topThreeUsers(aid, conn)
    if users != None:
        for i in range(len(users)):
            print(" | ".join(users[i]))

    # display top three playlists
    print("\n------Top Three Playlists------")
    playlists = topThreePlaylists(aid, conn)
    if playlists!= None:
        for i in range(len(playlists)):
            print(" | ".join(playlists[i]))

    input("press enter to continue")

def topThreeUsers(aid, conn):
    # find top three users

    c = conn.cursor()

    # query for top users
    getTopUsers = f"""
    SELECT u.*
    FROM songs s, listen l, perform p, users u
    WHERE s.sid = l.sid AND s.sid = p.sid
    AND p.aid = ?
    AND l.uid = u.uid
    GROUP BY l.uid
    ORDER BY SUM(s.duration * l.cnt) DESC
    LIMIT 3;
    """

    # execute query and return top users
    c.execute(getTopUsers, (aid,))
    topUsers = c.fetchall()
    return topUsers

def topThreePlaylists(aid, conn):
    # find top three playlists

    c = conn.cursor()

    # query for top playlists
    getTopPlaylists = f"""
    SELECT p.*
    FROM plinclude pl, perform pr, playlists p
    WHERE pl.sid = pr.sid AND p.pid = pl.pid
    AND pr.aid = ?
    GROUP BY pl.pid
    ORDER BY COUNT(pl.sid) DESC
    LIMIT 3
    """

    # execute query and return top playlists
    c.execute(getTopPlaylists, (aid,))
    topPlaylists = c.fetchall()
    topPlaylists = [list(i) for i in topPlaylists]
    topPlaylists = [[str(i[0])] + i[1:] for i in topPlaylists]
    return topPlaylists

def addSong(aid, conn):
    # add new song

    c = conn.cursor()
    print("------Add a new song------")

    # get song title and validate
    title = input("Enter the title of the song: ")
    if title == "":
        print("ERROR: title cannot be empty")
        input("press enter to continue")
        return

    # get duration and validate
    try:
        duration = int(input("Enter the duration: "))
    except:
        print("ERROR: duration must be an integer")
        input("press enter to continue")
        return

    # prompt the user to add or reject new song if title and duration are the same
    if not isNewSong(aid, title, duration, conn):
        print("Song with the same title and duration already exists")
        choice = input("Enter\n1 to proceed with adding the song\nAnything else to reject it\n")
        if choice!= '1':
            return

    # get any additional aids and check that aids do not conflict
    ids = [aid] + input("Enter the aids of any additional artists separated by spaces: ").split()
    if aid in ids[1:]:
        print("ERROR: cannot use own aid as additional aid")
        input("press enter to continue")
        return

    # Add new song into songs table
    sid = getUniqueSid(conn)

    insertSong = f"""
    INSERT INTO songs
    VALUES (?, ?, ?);
    """

    c.execute(insertSong, (sid, title, duration))

    # check that all aids are valid
    if False in [isValidAid(i, conn) for i in ids]:
        print("ERROR: you must enter valid aids")
        addPerformer(ids[0], sid, conn)
        input("press enter to continue")
        return

    # insert all aids into perform table
    for id in ids:
        addPerformer(id, sid, conn)

    conn.commit()
    input("press enter to continue")

def addPerformer(aid, sid, conn):
    # add new entry into perfrom table

    c = conn.cursor()
    insertPerform = f"""
    INSERT INTO perform
    VALUES (?, ?);
    """
    c.execute(insertPerform, (aid, sid))
    conn.commit()

def isValidAid(aid, conn):
    # check that any given aid is in the database

    c = conn.cursor()
    getArtist = f"""
    SELECT *
    FROM artists
    WHERE aid = ?;
    """

    c.execute(getArtist, (aid,))
    return bool(c.fetchone())

def getUniqueSid(conn):
    # get a unique sid different from all the ones in the database

    c = conn.cursor()
    sidQuery = """
    SELECT sid
    FROM songs
    """

    c.execute(sidQuery)
    sids = c.fetchall()
    if sids == []:
        return 1
    return max([i[0] for i in sids]) + 1

def isNewSong(aid, title, duration, conn):
    # check that any given song does not exist in the database

    c = conn.cursor()
    songQuery = f"""
    SELECT *
    FROM artists a, songs s, perform p
    WHERE a.aid = p.aid AND s.sid = p.sid
    AND title = ? AND duration = ?
    AND a.aid = ?;
    """

    c.execute(songQuery, (title, duration, aid))
    return not bool(c.fetchone())

if __name__ == "__main__":
    #db = sqlite3.connect("./new.db")
    #topThreePlaylists("a10", sqlite3.connect("./a2.db"))
    #print(isNewSong("a1", "Applause", 212, sqlite3.connect("./a2.db")))
    #print(type(getAllSids(sqlite3.connect("./new.db"))[0]))
    #print(isValidAid("a10", sqlite3.connect("./new.db")))
    #addSong("a10", sqlite3.connect("./new.db"))
    artistsMainMenu("a10", sqlite3.connect("./new.db"))
    #print(getUniqueSid(sqlite3.connect("./new.db")))
    pass


# Tai Sakuma <tai.sakuma@gmail.com>
import os
import importlib.util

##__________________________________________________________________||
def load_section_lists(cmsbook_path, chapter_path, thisfile):
    sections = _load(cmsbook_path, chapter_path)
    sections = _expand(sections, chapter_path, thisfile)
    return sections

##__________________________________________________________________||
def _load(cmsbook_path, chapter_path):
    path = os.path.join(cmsbook_path, chapter_path, 'cmsbook2_chapter', 'sections.py')
    spec = importlib.util.spec_from_file_location('sections', path)
    sections = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(sections)
    return sections.sections

##__________________________________________________________________||
def _expand(contents, thisdir, thisfile):
    contents = _copy_contents(contents)
    _add_href(contents, thisdir)
    _add_thisfile(contents, thisdir, thisfile)
    _add_thisfile_ancestor(contents)
    return contents

##__________________________________________________________________||
def _copy_contents(contents):
    """return a copy of contents

    This function is used instead of the very slow copy.deepcopy().
    """
    ret = [i.copy() for i in contents]
    for item in ret:
        if 'subcontents' in item:
            item['subcontents'] = _copy_contents(item['subcontents'])
    return ret

def _add_href(contents, thisdir):
    for item in contents:
        _add_href_item(item, thisdir)

def _add_href_item(item, thisdir):
    if 'href' not in item:
        if 'dir' in item:
            if 'file' in item:
                item['href'] = '/{}/{}/{}'.format(thisdir, item['dir'], item['file'])
            else:
                item['href'] = '/{}/{}/'.format(thisdir, item['dir'])
        elif 'file' in item:
                item['href'] = '/{}/{}'.format(thisdir, item['file'])

    if 'subcontents' in item:
        _add_href(item['subcontents'], thisdir)

def _add_thisfile(contents, thisdir, thisfile):
    for item in contents:
        _add_thisfile_item(item, thisdir, thisfile)

def _add_thisfile_item(item, thisdir, thisfile):
    if 'dir' in item and 'file' in item:
        if item['dir'] == thisdir and item['file'] == thisfile:
            item['thisfile'] = True

    if 'subcontents' in item:
        _add_thisfile(item['subcontents'], thisdir, thisfile)

def _add_thisfile_ancestor(contents):
    for item in contents:
        if item.get('thisfile', False):
            return True
        if 'subcontents' in item:
            if _add_thisfile_ancestor(item['subcontents']):
                item['thisfile_ancestor'] = True
                return True
    return False

##__________________________________________________________________||


import subprocess
import wolframalpha
import pyttsx3
import json
import operator
import speech_recognition as sr
import datetime
import wikipedia
import webbrowser
import os
import pyjokes
import feedparser
import time
import requests
import shutil
from bs4 import BeautifulSoup

class Ben:
    def __init__(self):
        self.engine = pyttsx3.init('sapi5')
        self.voices = self.engine.getProperty('voices')
        self.engine.setProperty('voice', self.voices[1].id)

        self.app_id = "Wolframalpha api id"
        self.client = wolframalpha.Client(self.app_id)

        self.wishMe()
        self.username()

    def speak(audio):
        self.engine.say(audio)
        self.engine.runAndWait()

    def wishMe():
        self.hour = int(datetime.datetime.now().hour)
        if self.hour>= 0 and hour<12:
            speak("Good Morning Sir !")

        elif self.hour>= 12 and hour<18:
            speak("Good Afternoon Sir !")

        else:
            speak("Good Evening Sir !")

        self.ben = "Ben"
        speak("I am your Assistant")
        speak(self.ben)

    def username():
        speak("What should i call you sir")
        self.uname = takeCommand()
        speak("Welcome Mister")
        speak(self.uname)
        self.columns = shutil.get_terminal_size().columns

        print("#####################".center(self.columns))
        print("Welcome Mr.", self.uname.center(self.columns))
        print("#####################".center(self.columns))

        speak("How can I help you, Sir")

    def takeCommand():

        self.r = sr.Recognizer()

        with sr.Microphone() as source:

            print("Listening...")
            self.r.pause_threshold = 1
            audio = self.r.listen(source)

        try:
            print("Recognizing...")
            self.query = self.r.recognize_google(audio, language ='en-in')
            print(f"User said: {self.query}\n")

        except Exception as e:
            print(e)
            print("Unable to Recognize your voice.")
            return "None"

        return self.query

    def main(self):
        self.query = takeCommand().lower()

        if "Ben" in self.query and "wikipedia" in self.query:
            speak('Searching Wikipedia...')
            self.query = self.query.replace("wikipedia", "")
            self.results = wikipedia.summary(self.query, sentences = 3)
            speak("According to Wikipedia")
            print(self.results)
            speak(self.results)

        elif 'Ben' in self.query and "youtube" in self.query:
            speak("Here you go to Youtube\n")
            webbrowser.open("youtube.com")

        elif "Ben" in self.query and "time" in self.query:
            self.strTime = datetime.datetime.now().strftime("% H:% M:% S")
            speak(f"Sir, the time is {self.strTime}")

        elif "Ben" in self.query and "joke" in self.query:
            speak(pyjokes.get_joke())

        elif "Ben" in self.query and "News" in self.query:
            pass
            # not done
            #except Exception as e:
            #    print(str(e))

        elif "Ben where is" in self.query:
            self.query = self.query.replace("where is", "")
            self.location = self.query
            speak("User asked to Locate")
            speak(self.location)
            webbrowser.open("https://www.google.nl / maps / place/" + self.location + "")

        elif "Ben write a note" in self.query:
            speak("What should i write, sir")
            self.note = takeCommand()
            self.file = open('jarvis.txt', 'w')
            speak("Sir, Should i include date and time")
            self.snfm = takeCommand()
            if 'yes' in snfm or 'sure' in snfm:
                self.strTime = datetime.datetime.now().strftime("% H:% M:% S")
                self.file.write(self.strTime)
                self.file.write(" :- ")
                self.file.write(self.note)
            else:
                self.file.write(self.note)

        elif "Ben" in self.query and "show note" in self.query:
            speak("Showing Notes")
            self.file = open("jarvis.txt", "r")
            print(self.file.read())
            speak(self.file.read(6))

        elif "Ben" in self.query and "weather" in self.query:

            # Google Open weather website
            # to get API of Open weather
            api_key = "Api key"
            base_url = "http://api.openweathermap.org / data / 2.5 / weather?"
            speak(" City name ")
            print("City name : ")
            city_name = takeCommand()
            complete_url = base_url + "appid =" + api_key + "&q =" + city_name
            response = requests.get(complete_url)
            x = response.json()

            if x["code"] != "404":
                y = x["main"]
                current_temperature = y["temp"]
                current_pressure = y["pressure"]
                current_humidiy = y["humidity"]
                z = x["weather"]
                weather_description = z[0]["description"]
                print(" Temperature (in kelvin unit) = " +str(current_temperature)+"\n atmospheric pressure (in hPa unit) ="+str(current_pressure) +"\n humidity (in percentage) = " +str(current_humidiy) +"\n description = " +str(weather_description))

            else:
                speak(" City Not Found ")

        elif "send message " in self.query:
                # You need to create an account on Twilio to use this service
                account_sid = 'Account Sid key'
                auth_token = 'Auth token'
                client = Client(account_sid, auth_token)

                message = client.messages \
                                .create(
                                    body = takeCommand(),
                                    from_='Sender No',
                                    to ='Receiver No'
                                )

                print(message.sid)

        elif "i love you" in self.query:
            speak("that's kind of gay not going to lie")

        elif "what is" in self.query or "who is" in self.query:
            self.res = client.self.query(self.query)

            try:
                print (next(self.res.results).text)
                speak (next(self.res.results).text)
            except StopIteration:
                print ("No results")

        elif "Ben calculate" in self.query:
            self.indx = self.query.lower().split().index('calculate')
            self.query = self.query.split()[self.indx + 1:]
            self.res = self.client.self.query(' '.join(self.query))
            self.answer = next(self.res.results).text
            print("The answer is " + self.answer)
            speak("The answer is " + self.answer)


from PIL import Image, ImageDraw, ImageFont
import random

def plural(action):
    if len(action.split()) > 1:
        return ' '.join([f"{action.split()[0]}s", action.split()[1]])
    else:
        return f"{action}s"

def random_phrase() -> list:
    nouns = open("nouns.txt", "r").readlines()
    actions = open("actions.txt", "r").readlines()

    ret = f"When the {random.choice(nouns)[:-1]} {plural(random.choice(actions)[:-1])} you"

    return [[(0, 0), ret]]

def random_nouns(count, x, spacing) -> list:
    nouns = open("nouns.txt", "r").readlines()

    return [[(x, spacing * i), random.choice(nouns)] for i in range(count)]

def generate_meme(meme) -> list:
    verb = random.choice(open('verbs.txt', 'r').readlines())[:-1]
    noun = random.choice(open('nouns.txt', 'r').readlines())[:-1]
    noun2 = random.choice(open('nouns.txt', 'r').readlines())[:-1]
    action = random.choice(open('actions.txt', 'r').readlines())[:-1]
    noun3 = random.choice(open('nouns.txt', 'r').readlines())[:-1]
    adjective = random.choice(open('adjs.txt', 'r').readlines())[:-1]

    texts = {
        "peace": random_phrase(),
        "uno": [[(100, 150), f"{verb} {noun}s"], [(300, 20), f"{noun2}s"]],
        "incredible": [[(50, 150), f"Me when I {action} the {noun}"], [(600, 150), f"It's {adjective}"]],
        "joey": [[(0, 0), f"Me when I {action} the {noun}"]],
        "crying": [[(0, 0), f"Me when I {action} the {adjective} {noun}"]],
        "monkey": [[(0, 0), f"Me when I {action} the {noun}"]]
    }

    return texts[meme]

def main():
    meme = random.choice(["peace", "uno", "incredible", "joey", "crying", "monkey"])
    caption = generate_meme(meme)

    img = Image.open(f"formats/{meme}.png")
    draw = ImageDraw.Draw(img)
    font = ImageFont.truetype('impact.ttf', size=24)
    print(caption)

    for l in caption:
        print(l[0])
        draw.text((l[0][0] - 1, l[0][1] - 1), l[1], 'black', font)
        draw.text((l[0][0] + 1, l[0][1] - 1), l[1], 'black', font)
        draw.text((l[0][0] - 1, l[0][1] + 1), l[1], 'black', font)
        draw.text((l[0][0] + 1, l[0][1] + 1), l[1], 'black', font)

        draw.text(l[0], l[1], 'white', font)

    img.save('out.png')

    with open("caption", "w") as f:
        for i in caption:
            f.write(i[1])
            f.write('.\n')

if __name__ == '__main__':
    main()


import sys, json, ctypes, os, math, shutil, subprocess, time
import win32com.client
from pathlib import Path
from szipper import szip

def pretty(tgt='', sym='-', length=55):
	length = length - 2
	result = ''
	result += math.ceil((length - len(tgt))/2)*sym
	result += ' '
	result += tgt
	result += ' '
	result += math.floor((length - len(tgt))/2)*sym
	return result

def drop(amt=1):
	for d in range(amt):
		print('')

drop()
print("""
=======================================================
=======================================================
              KICKBOXER 3000 INSTALLING
=======================================================
=======================================================
""")
drop(5)

def main():

	print(pretty('Spinning Earth...'))

	drop()
	print(pretty('Discovering LV-426...'))

	bin_root = Path(__file__).parent
	exe_dir = Path(sys.executable).parent

	drop()
	print(pretty('Sending Chuck Norris...'))

	szipper = szip(bin_root / '7z.exe')

	# ========================================
	#   Unpack controller to the current dir
	# ========================================
	szipper.unpack(
		bin_root / 'kb_data.7z',
		exe_dir,
	)

	drop(4)
	print(pretty('Mission Status: No survivors...'))
	drop(1)

	# ========================================
	# Move 7zip executable to the bins folder inside controller
	# ========================================
	(exe_dir / 'KickBoxer3000-win32-x64' / 'resources' / 'app' / 'src' / 'bins' / '7z').mkdir(parents=True, exist_ok=True)
	shutil.move(
		bin_root / '7z.exe',
		exe_dir / 'KickBoxer3000-win32-x64' / 'resources' / 'app' / 'src' / 'bins' / '7z' / '7z.exe',
	)
	shutil.move(
		bin_root / '7z.dll',
		exe_dir / 'KickBoxer3000-win32-x64' / 'resources' / 'app' / 'src' / 'bins' / '7z' / '7z.dll',
	)

	# ========================================
	#       Important: create a lock file
	# ========================================
	(exe_dir / 'KickBoxer3000-win32-x64' / 'resources' / 'app' / 'src' / 'raw_base_is.bad').write_text('Achtung')

	# ========================================
	#            Create a shortcut
	# ========================================
	shotcut_loc = str(exe_dir / 'KickBoxer3000.lnk')
	shotcut_target = str(exe_dir / 'KickBoxer3000-win32-x64' / 'KickBoxer3000.exe')
	# icon = r'C:\path\to\icon\resource.ico'

	shell = win32com.client.Dispatch('WScript.Shell')
	shortcut = shell.CreateShortCut(shotcut_loc)
	shortcut.Targetpath = shotcut_target
	# shortcut.IconLocation = icon
	# shortcut.WindowStyle = 7
	shortcut.save()

	print(pretty('... among Aliens.'))
	drop(4)

	# ========================================
	#            Reload Icons
	# ========================================
	subprocess.call([r'C:\Windows\System32\ie4uinit.exe', '-show'])

	print(pretty('Installation Complete. No Errors.'))
	drop(5)

	time.sleep(5)

if __name__ == '__main__':
	main()


import requests
from pprint import pprint

import datetime
import time
import csv
import json
import os

# start time
start_time = datetime.datetime.now()

# read questions form a file
file_question_topics = open("topic_list_answers.txt", mode="r", encoding="utf-8")
topics = file_question_topics.readlines()

def get_voteCount(contents, upvote_index):
    end_index = []
    for index in upvote_index:
        end = index
        while contents[end] != "," and contents[end] != "}":
            end +=1
        end_index.append(end)
    return end_index

def save_data_as_json(data, path,ensure_ascii=True):
    with open(path, "w") as f:
        json.dump(data, f, ensure_ascii=ensure_ascii)

for topic in topics:
    quora_question_answers = {}
    topic = topic.replace("\n","")
    quora_question_answers[topic] = []

    questions_list = open("./questions/" + topic + ".txt", "r",encoding="utf-8")
    urls = questions_list.readlines()

    total = len(urls)

    count = 0
    for url in urls:
        question_answer_pair = {}
        question_answer_pair["url"] = url.replace('\n','')
        question_answer_pair["question"] = url.replace("https://www.quora.com/","").replace("-"," ").replace('\n','')
        res = requests.get(url)
        contents = res.text

        text_index = []
        upvote_index = []

        for i in range(len(contents)-14):
            text_flag = contents[i:i+7]
            upvote_flag = contents[i:i+14]
            url_flag = contents[i:i+6]

            if text_flag == '"text":':
                text_index.append(i+7)
            if upvote_flag == '"upvoteCount":':
                upvote_index.append(i+14)

        end_index = get_voteCount(contents, upvote_index)

        answers = []
        for i in range(len(text_index)-1):
            answer = {}
            answer["upvote"] = int(contents[upvote_index[i]+1:end_index[i]])
            answer["content"] = contents[text_index[i]+2:upvote_index[i]-17]
            answers.append(answer)

        question_answer_pair["answers"] = answers
        quora_question_answers[topic].append(question_answer_pair)

        count +=1
        if count%20 ==0:
            print(str(count) + "/"+ str(total) + " "+ str(datetime.datetime.now() - start_time))

    # save data as json format
    path = "./results/" + topic + ".json"
    save_data_as_json(quora_question_answers,path)

# display processing time
end_time = datetime.datetime.now()
print(end_time - start_time)


import matplotlib.pyplot as plt

line_lyr = QgsProject.instance().mapLayersByName('New scratch layer')[0]
rl = QgsProject.instance().mapLayersByName('S17E130')[0]

line_geom = [f.geometry() for f in line_lyr.getFeatures()][0]

line_length = line_geom.length()

num_profile_pts = 25

interp_dist = line_length/num_profile_pts

total_dist = interp_dist

sample_geoms = []

while total_dist < line_length:
    sample_geoms.append(line_geom.interpolate(total_dist))
    total_dist+=interp_dist

#print(sample_pts)

plot_vals = [rl.dataProvider().sample(geom.asPoint(), 1)[0] for geom in sample_geoms]

#print(plot_vals)

plt.plot(plot_vals, label='Profile', color='brown', linewidth=2)

plt.show()


import argparse
import asyncio
import time
from concurrent.futures import ThreadPoolExecutor, wait, ALL_COMPLETED

import syft
from syft.workers.websocket_client import WebsocketClientWorker
import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
from torch.utils.data import TensorDataset, DataLoader, Dataset
from torchvision import datasets, transforms
import numpy as np
from PIL import Image

from goodDataML.learning.testnet.utils import get_do_uuid_from_info, get_mpc_uuid_from_info
from goodDataML.connection.chain import GoodDataChain, ChainEvent

class MNISTDataset(Dataset):
    def __init__(self, data, targets, transform):
        self.data = data
        self.targets = targets
        self.transform = transform

    def __len__(self):
        return len(self.data)

    def __getitem__(self, index):
        """
        Args:
            index (int): Index
        Returns:
            tuple: (image, target) where target is index of the target class.
        """
        img, target = self.data[index], int(self.targets[index])

        # doing this so that it is consistent with all other datasets
        # to return a PIL Image
        img = Image.fromarray(img.numpy(), mode='L')

        if self.transform is not None:
            img = self.transform(img)

        return img, target

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(1, 32, 3, 1)
        self.conv2 = nn.Conv2d(32, 64, 3, 1)
        self.dropout1 = nn.Dropout2d(0.25)
        self.dropout2 = nn.Dropout2d(0.5)
        self.fc1 = nn.Linear(9216, 128)
        self.fc2 = nn.Linear(128, 10)

    def forward(self, x):
        x = self.conv1(x)
        x = F.relu(x)
        x = self.conv2(x)
        x = F.relu(x)
        x = F.max_pool2d(x, 2)
        x = self.dropout1(x)
        x = torch.flatten(x, 1)
        x = self.fc1(x)
        x = F.relu(x)
        x = self.dropout2(x)
        x = self.fc2(x)
        output = F.log_softmax(x, dim=1)
        return output

def update(data: syft.PointerTensor, target: syft.PointerTensor, model, optimizer):
    """
    train one batch data for DO
    :param data
    :param target
    :param model
    :param optimizer optimizer is a optimizer function in torch.optim lib
    :return updated model
    """
    model.send(data.location)
    optimizer.zero_grad()
    output = model(data)
    loss = F.nll_loss(output, target)
    loss.backward()
    optimizer.step()
    return model

async def share_param(remote_index: int, params: list,
                      mpc_nodes: tuple):
    """
    async function, DO share param to MPC nodes
    :param remote_index: DO's location in list of DOs
    :param params
    :param mpc_nodes
    :return
    """
    params_remote_index = list()
    for param_i in range(len(params[0])):
        params_remote_index.append(
            params[remote_index][param_i].fix_precision().get().share(
                *mpc_nodes,
                crypto_provider=None
            ).get())
    return params_remote_index

def predict(chain: GoodDataChain,
            prediction_uuid: str,
            model):
    """
    test the trained model
    :param chain
    :param prediction_uuid
    :param model
    :return
    """
    # set logs
    prediction_logs = dict()
    prediction_logs["start_time"] = int(time.time())

    transform = transforms.Compose([
        transforms.ToTensor(),
        transforms.Normalize((0.1307,), (0.3081,))
    ])
    kwargs = {'batch_size': 64}
    dataset2 = datasets.MNIST('../data', train=False,
                              transform=transform)
    test_loader = torch.utils.data.DataLoader(dataset2, **kwargs)

    model.eval()
    test_loss = 0
    correct = 0
    with torch.no_grad():
        for data, target in test_loader:
            output = model(data)
            test_loss += F.nll_loss(output, target, reduction='sum').item()  # sum up batch loss
            pred = output.argmax(dim=1, keepdim=True)  # get the index of the max log-probability
            correct += pred.eq(target.view_as(pred)).sum().item()

    test_loss /= len(test_loader.dataset)

    print('\nPredict set: Average loss: {:.4f}, Accuracy: {}/{} ({:.4f}%)\n'.format(
        test_loss, correct, len(test_loader.dataset),
        100. * correct / len(test_loader.dataset)))
    prediction_logs["loss"] = test_loss
    prediction_logs["accuracy"] = 1. * correct / len(test_loader.dataset)
    prediction_logs["end_time"] = int(time.time())
    chain.add_log(prediction_uuid, prediction_logs)

def validate(chain: GoodDataChain,
             epoch: int,
             total_epochs: int,
             query_uuid: str,
             query_info_and_nodes_info,
             models: list,
             test_loader: torch.utils.data.DataLoader):
    """
    validate the model
    :param chain
    :param epoch
    :param total_epochs
    :param query_uuid
    :param query_info_and_nodes_info
    :param models
    :param test_loader
    :return
    """
    # set logs
    test_logs = dict()
    test_logs["task"] = "validation"
    test_logs["epoch"] = epoch + 1
    test_logs["total_epochs"] = total_epochs
    test_logs["start_time"] = int(time.time())

    models[0].eval()
    test_loss = 0
    correct = 0
    with torch.no_grad():
        for data, target in test_loader:
            output = models[0](data)
            test_loss += F.nll_loss(output, target, reduction='sum').item()  # sum up batch loss
            pred = output.argmax(dim=1, keepdim=True)  # get the index of the max log-probability
            correct += pred.eq(target.view_as(pred)).sum().item()

    test_loss /= len(test_loader.dataset)

    print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.4f}%)\n'.format(
        test_loss, correct, len(test_loader.dataset),
        100. * correct / len(test_loader.dataset)))
    test_logs["loss"] = test_loss
    test_logs["accuracy"] = 1. * correct / len(test_loader.dataset)
    test_logs["end_time"] = int(time.time())
    chain.add_log(query_uuid, test_logs)

def define_and_get_arguments():
    """
    Define the hyperparameters required, features and target for training
    :return dic
    """
    parser = argparse.ArgumentParser(
        description="Run federated learning using websocket client workers."
    )
    parser.add_argument("--batch_size", type=int, default=64, help="batch size of the training")
    parser.add_argument("--test_batch_size", type=int, default=1000, help="batch size of test")
    parser.add_argument("--epochs", type=int, default=5, help="number of epochs to train")
    parser.add_argument("--lr", type=float, default=1.0, help="learning rate")
    parser.add_argument("--cuda", action="store_true", help="use cuda")
    parser.add_argument("--target", type=str, default='Survived', help="training target")
    parser.add_argument(
        "--verbose",
        "-v",
        action="store_true",
        help="if set, websockets client workers will " "be started in verbose mode",
    )
    args = parser.parse_args(args=[])
    return args

def send_data_to_worker(data: np.array,
                        target: np.array,
                        transform: transforms,
                        compute_node: WebsocketClientWorker,
                        batch_size: int):
    """
    send data to websocket compute node
    :param data: numpy.array
    :param target
    :param transform
    :param compute_node
    :param batch_size
    :return list
    """
    remote_dataset = []
    train_dataset = MNISTDataset(data, target, transform)
    train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)

    for batch_idx, (data, target) in enumerate(train_loader):
        data = data.send(compute_node)
        target = target.send(compute_node)
        remote_dataset.append((data, target))
    return remote_dataset

def get_remote_dataset_and_validation_dataloader(batch_size: int, compute_nodes: list, data_set: int):
    """
    1) get training data and send to remote DO
    2) get test DataLoader
    :param batch_size:
    :param compute_nodes:
    :param data_set:
    :return
    """
    transform = transforms.Compose([
        transforms.ToTensor(),
        transforms.Normalize((0.1307,), (0.3081,))
    ])
    kwargs = {'batch_size': batch_size}

    dataset1 = datasets.MNIST('../data', train=True, download=True,
                              transform=transform)

    # get remote datasets
    print("data_set = ", data_set)
    if data_set == 1:
        size = 1000
    else:
        size = 10000
    print("dataset size: ", size)
    data = dataset1.data[:size]
    targets = dataset1.targets[:size]
    training_data_size = int(len(data) / len(compute_nodes))
    remote_datasets = []
    left = 0
    right = training_data_size
    for i in range(len(compute_nodes)):
        remote_datasets.append(send_data_to_worker(
            data[left:right], targets[left:right], transform, compute_nodes[i], batch_size))
        left = right
        right += training_data_size

    # test Dataloader
    validation_data = dataset1.data[50000:]
    validation_target = dataset1.targets[50000:]
    validation_dataset = MNISTDataset(validation_data, validation_target, transform)
    test_loader = torch.utils.data.DataLoader(validation_dataset, **kwargs)

    return remote_datasets, test_loader

async def train(chain: GoodDataChain,
                epoch: int,
                total_epochs: int,
                query_uuid: str,
                query_info_and_nodes_info: dict,
                compute_nodes: list,
                remote_dataset: list,
                models: list,
                optimizers: list,
                params: list,
                mpc_nodes: tuple):
    """
    Realize multi-party asynchronous federated learning
    :param chain: write a journal to the chain
    :param epoch
    :param total_epochs
    :param query_uuid
    :param query_info_and_nodes_info
    :param compute_nodes
    :param remote_dataset
    :param models: Multi-party model
    :param optimizers
    :param params: model parameters
    :param mpc_nodes
    :return models
    """

    do_uuids = get_do_uuid_from_info(query_info_and_nodes_info)
    mpc_uuids = get_mpc_uuid_from_info(query_info_and_nodes_info)
    for data_index in range(len(remote_dataset[0])):
        print("batch_index: ", data_index)
        # we encrypt it on the remote machine
        pool = ThreadPoolExecutor(max_workers=10)
        tasks = []
        train_logs = dict()
        train_logs["task"] = "train"
        train_logs["epoch"] = epoch
        train_logs["total_epochs"] = total_epochs
        train_logs["batch_index"] = data_index
        train_logs["start_time"] = int(time.time())
        train_logs["do_uuid"] = do_uuids
        train_logs["mpc_uuid"] = mpc_uuids

        for remote_index in range(len(compute_nodes)):
            data, target = remote_dataset[remote_index][data_index]
            tasks.append(pool.submit(update, data, target,
                                     models[remote_index], optimizers[remote_index]))

        # wait all DOs to finish training
        wait(tasks, return_when=ALL_COMPLETED)
        for remote_index in range(len(compute_nodes)):
            models[remote_index] = tasks[remote_index].result()

        # encrypted aggregation
        new_params = list()
        tasks = [asyncio.create_task(share_param(remote_index,
                                                 params,
                                                 mpc_nodes,
                                                 ))
                 for remote_index in range(len(compute_nodes))]
        await asyncio.wait(tasks, return_when=asyncio.ALL_COMPLETED)
        new_params_list = list()
        for task in tasks:
            new_params_list.append(task.result())

        for param_i in range(len(params[0])):
            new_params.append(
                sum(new_params_list[remote_index][param_i]
                    for remote_index in range(len(compute_nodes))).
                float_precision() / len(compute_nodes)
            )
        train_logs["total_batch_indexes"] = len(remote_dataset[0])
        train_logs["end_time"] = int(time.time())
        # add_logs_to_chain(chain, query_uuid, train_logs)
        chain.add_log(query_uuid, train_logs)
        # clean up
        with torch.no_grad():
            for model_param in params:
                for param in model_param:
                    param = param.get()
                    param *= 0

            for remote_index in range(len(compute_nodes)):
                for param_index in range(len(params[remote_index])):
                    params[remote_index][param_index].set_(new_params[param_index])

    return models

def train_model(chain: GoodDataChain,
                query_uuid: str,
                query_info_and_nodes_info: dict,
                compute_nodes: list,
                model,
                mpc_nodes: tuple):
    """
    The main process of model training
    :param chain
    :param query_uuid
    :param query_info_and_nodes_info
    :param compute_nodes
    :param model
    :param mpc_nodes
    :return model: Trained model
    """
    print("start training.")
    args = define_and_get_arguments()
    remote_datasets, test_loader = get_remote_dataset_and_validation_dataloader(
                                        args.batch_size,
                                        compute_nodes,
                                        query_info_and_nodes_info.query_info.data_set
                                    )

    models = []
    for i in range(len(compute_nodes)):
        models.append(model.copy())

    params = [list(compute_node_model.parameters()) for compute_node_model in models]
    optimizers = [optim.Adadelta(compute_node_model.parameters(), lr=args.lr)
                  for compute_node_model in models]

    # training and test
    print("start time: ", time.strftime("%X"))

    for epoch in range(args.epochs):
        print(f"Epoch {epoch + 1}")
        asyncio.run(train(chain,
                          epoch,
                          args.epochs,
                          query_uuid,
                          query_info_and_nodes_info,
                          compute_nodes,
                          remote_datasets,
                          models,
                          optimizers,
                          params,
                          mpc_nodes))
        validate(chain, epoch, args.epochs, query_uuid, query_info_and_nodes_info, models, test_loader)

    print("end time: ", time.strftime("%X"))

    # Close websocket connection
    for compute_node in compute_nodes:
        if isinstance(compute_node, WebsocketClientWorker):
            compute_node.close()

    for mpc_node in mpc_nodes:
        if isinstance(mpc_node, WebsocketClientWorker):
            mpc_node.close()

    return models[0]


from __future__ import annotations

import asyncio
import datetime
import unittest

from tvaf import ltpy
from tvaf import services

from . import lib

class FormatTest(lib.AppTest, lib.TestCase):
    async def test_invalid_info_hash(self) -> None:
        r = await self.client.get("/torrents/a")
        self.assertEqual(r.status_code, 422)
        self.assert_golden_json(r.json(), suffix="short.json")

        r = await self.client.get("/torrents/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
        self.assertEqual(r.status_code, 422)
        self.assert_golden_json(r.json(), suffix="long.json")

        r = await self.client.get("/torrents/zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz")
        self.assertEqual(r.status_code, 422)
        self.assert_golden_json(r.json(), suffix="not_hex.json")

class StatusTest(lib.AppTestWithTorrent, lib.TestCase):
    @unittest.skip("flaky")
    async def test_get(self) -> None:
        r = await self.client.get(f"/torrents/{self.torrent.sha1_hash}")
        self.assertEqual(r.status_code, 200)
        # some fields are unstable, so content-length is unstable
        headers = dict(r.headers)
        headers.pop("content-length")
        self.assert_golden_json(headers, suffix="headers.json")

        status_dict = r.json()
        # test unstable parts
        self.assertIsInstance(status_dict.pop("added_time"), int)
        self.assertIsInstance(status_dict.pop("completed_time"), int)
        datetime.datetime.fromisoformat(status_dict.pop("last_download"))
        self.assertEqual(status_dict.pop("save_path"), self.tempdir.name)
        self.assert_golden_json(status_dict, suffix="body.json")

class GetPiecePrioritiesTest(lib.AppTestWithTorrent, lib.TestCase):
    async def test_get(self) -> None:
        r = await self.client.get(
            f"/torrents/{self.torrent.sha1_hash}/piece_priorities"
        )
        self.assertEqual(r.status_code, 200)
        self.assert_golden_json(dict(r.headers), suffix="headers.json")
        self.assert_golden_json(r.json(), suffix="body.json")

class RemoveTest(lib.AppTestWithTorrent, lib.TestCase):
    @unittest.skip("flaky")
    async def test_delete(self) -> None:
        r = await self.client.delete(f"/torrents/{self.torrent.sha1_hash}")
        self.assertEqual(r.status_code, 200)
        self.assertFalse(
            await asyncio.to_thread(
                ltpy.handle_in_session,
                self.handle,
                await services.get_session(),
            )
        )

class TorrentListTest(lib.AppTestWithTorrent, lib.TestCase):
    @unittest.skip("flaky")
    async def test_torrent_list(self) -> None:
        r = await self.client.get("/torrents")
        self.assertEqual(r.status_code, 200)
        # some fields are unstable, so content-length is unstable
        headers = dict(r.headers)
        headers.pop("content-length")
        self.assert_golden_json(headers, suffix="headers.json")

        statuses = r.json()
        # test unstable parts
        self.assertIsInstance(statuses[0].pop("added_time"), int)
        self.assertIsInstance(statuses[0].pop("completed_time"), int)
        datetime.datetime.fromisoformat(statuses[0].pop("last_download"))
        self.assertEqual(statuses[0].pop("save_path"), self.tempdir.name)
        self.assert_golden_json(statuses, suffix="body.json")