import numpy as np
import cv2
from PIL import Image
import os
import sys
sys.path.append(os.pardir)
from starter_code.utils import load_case
import shutil
# cd /data/yangkaixing/kits19/data_statistic
# nohup python get_data.py > get_data.log 2>&1 &

people_num = 210
divide_num = int(210 * 0.7)
root_train_img_path = '/data/yangkaixing/kits19/Main/data/images/train'
root_train_label_path = '/data/yangkaixing/kits19/Main/data/labels/train'
root_test_img_path = '/data/yangkaixing/kits19/Main/data/images/test'
root_test_label_path = '/data/yangkaixing/kits19/Main/data/labels/test'

shutil.rmtree(root_train_img_path, ignore_errors=True)
shutil.rmtree(root_train_label_path, ignore_errors=True)
shutil.rmtree(root_test_img_path, ignore_errors=True)
shutil.rmtree(root_test_label_path, ignore_errors=True)
os.makedirs(root_train_img_path, exist_ok=True)
os.makedirs(root_train_label_path, exist_ok=True)
os.makedirs(root_test_img_path, exist_ok=True)
os.makedirs(root_test_label_path, exist_ok=True)

train_txt = '/data/yangkaixing/kits19/Main/data/train.txt'
test_txt = '/data/yangkaixing/kits19/Main/data/test.txt'
train_f = open(train_txt, 'w')
test_f = open(test_txt, 'w')

for i in range(0, people_num):

    volume, segmentation = load_case(i)
    seg_data = np.array(segmentation.get_fdata())
    img_data = np.array(volume.get_fdata())

    for j in range(seg_data.shape[0]):

        kidney_idx = seg_data[j] == 1
        kidney_exist = len(seg_data[j][kidney_idx]) > 0

        if kidney_exist > 0: # 我们只把存在肾的数据图片提取出来

            if i < divide_num:

                # train_img_path = os.path.join(root_train_img_path, str(i))
                # train_label_path = os.path.join(root_train_label_path, str(i))
                # os.makedirs(train_img_path, exist_ok=True)
                # os.makedirs(train_label_path, exist_ok=True)
                img_name = os.path.join(root_train_img_path, '{:0>4d}_{:0>4d}.png'.format(i, j))
                label_name = os.path.join(root_train_label_path, '{:0>4d}_{:0>4d}.npy'.format(i, j))
                train_f.write(img_name + ' ' + label_name + '\n')

            else:

                # test_img_path = os.path.join(root_test_img_path, str(i))
                # test_label_path = os.path.join(root_test_label_path, str(i))
                # os.makedirs(test_img_path, exist_ok=True)
                # os.makedirs(test_label_path, exist_ok=True)
                img_name = os.path.join(root_test_img_path, '{:0>4d}_{:0>4d}.png'.format(i, j))
                label_name = os.path.join(root_test_label_path, '{:0>4d}_{:0>4d}.npy'.format(i, j))
                test_f.write(img_name + ' ' + label_name + '\n')

            img = Image.fromarray(img_data[j]).convert('RGB')
            img.save(img_name)

            one_hot_values= np.eye(3)[seg_data[j].astype(np.uint8)]
            label = np.uint8(one_hot_values)
            np.save(label_name, label)
            # label = (seg_data[j] * (255.0 / 2)).astype(np.uint8)
            # label = Image.fromarray(label)
            # label.save(label_name)

train_f.close()
test_f.close()


import argparse
import utils
import numpy as np
import datetime
from ner_config import Config
from models.ner_bilstm import RNN

def evaluate(model, test_set, config):
    '''
        Evaluates a test set

        Input: model, dataset to evaluate, config object
        Ouput: dictionary containing data about the evaluation

    '''
    # Build a confusion matrix
    matrix = [[0 for x in range(len(config.tags))]for y in range(len(config.tags))]
    # Pad and get sequence_lengths and word_lengths if necessary
    words, tags, chars = config.pad(dataset=test_set, use_chars=config.use_chars)
    # Create the feed dictionary
    feed = config.feed(model, words, tags, chars, use_chars=config.use_chars, dropout=1.0)
    # Get the bacth prediction , shape = (batch_size,max_sentence_length)
    sentences = model.predict_batch(feed)
    # for each line
    for pred, label in zip(sentences, tags):
        for i in range(len(pred)):
            matrix[pred[i]][label[i]] += 1

    metrics = dict()
    # Total labels
    metrics["total"] = np.sum(matrix)
    # Save the matrix
    metrics["matrix"] = np.array(matrix)
    # Number of real labels in the test set
    real_labels = np.sum(matrix, axis=0)
    # Number of predicted labels in the test set
    pred_labels = np.sum(matrix, axis=1)
    # Array of correct labels
    correct_labels = np.diagonal(matrix)
    # Total number of correct labels
    total_correct = np.sum(correct_labels)
    # Disable division zero by zero warnings
    # This is due to there being no instances of a certain label
    np.seterr(invalid='ignore')
    # Accuracy
    metrics["accuracies"] = correct_labels / real_labels
    metrics["accuracy"] = (total_correct/metrics["total"])
    # Precision is out of the predicted labels, which ones were correct
    metrics["precisions"] = np.divide(correct_labels, pred_labels)
    # Remove any NaN due to some labels not being present in the test set
    metrics["precisions"][np.isnan(metrics["precisions"])] = 1
    # Get the average
    metrics["precision"] = np.average(metrics["precisions"])
    # Precision is out of the predicted labels, which ones were correct
    metrics["recalls"] = np.divide(correct_labels, real_labels)
    # Remove any NaN due to some labels not being present in the test set
    metrics["recalls"][np.isnan(metrics["recalls"])] = 0
    # Get the average
    metrics["recall"] = np.average(metrics["recalls"])
    # F1
    metrics["f1"] = (2 * metrics["precision"]*metrics["recall"]) / \
        (metrics["precision"]+metrics["recall"])
    metrics["f1s"] = (2 * metrics["precisions"]*metrics["recalls"]) / \
        (metrics["precisions"]+metrics["recalls"])

    return metrics

def getstats(config, results):
    # Initialize
    folds = len(results)
    n_sets = len(results[0])
    acc = [0] * n_sets
    f1 = [0] * n_sets
    precision = [0] * n_sets
    recall = [0] * n_sets
    line = datetime.datetime.now().strftime("%y%m%d%H%M%S") + "\t"
    # Collect
    for f in results:
        for i, dset in enumerate(f):
            acc[i] += dset["accuracy"]
            f1[i] += dset["f1"]
            precision[i] += dset["precision"]
            recall[i] += dset["recall"]
    # Compute
    acc = np.array(acc) / folds
    f1 = np.array(f1) / folds
    precision = np.array(precision) / folds
    recall = np.array(recall) / folds
    # Display
    for i in range(n_sets):
        # Add the metrics
        line += "%s\t%s\t%s\t%s\t" % (format((f1[i]), ".5f"), format(
            acc[i], ".5f"), format(precision[i], ".5f"), format(recall[i], ".5f"))
        print("\nTest set", i + 1, ":\n=================================\n",
              "RESULTS:\n\t Avg F1: \t", format(f1[i] * 100, '.3f'),
              "\n\t Avg Accuracy:  ", format(acc[i] * 100, '.3f'),
              "\n\t Avg Precision: ", format(precision[i] * 100, '.3f'),
              "\n\t Avg Recall: \t", format(recall[i] * 100, '.3f'),
              "\n=================================")
    # Add the config
    line += "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t" % (
        config.pretrained, config.dim_word, config.non_static,
        config.use_crf, config.cell_word, config.dropout, config.use_chars,
        config.dim_char, config.cell_char)
    return line

def classify(text, model, config):
    # Convert into IDs
    dataset = []
    for line in text:
        ids = []
        line = line.strip().split(" ")
        for w in line:
            ids.append(config.word_id(w))
        dataset.append(ids)
    # Pad and get sequence_lengths and word_lengths if necessary
    words, _, chars = config.pad(dataset=dataset, use_chars=config.use_chars, labels=False)
    # Create the feed dictionary
    feed = config.feed(model, words, tags=False, chars=chars,
                       use_chars=config.use_chars, dropout=1.0)
    # Get the bacth prediction , shape = (batch_size,max_sentence_length)
    if config.use_crf:
        predictions = model.predict_batch(feed)
    else:
        logits = model.logits.eval(feed, session=model.sess)
        predicitons = np.argmax(labels_pred, axis=-1)
    # return predictions
    return predictions

def NER_print(predictions, text, config):
    line_text = ""
    line_tags = ""
    for pred, word in zip(predictions, text):
        pred = config.id_to_tag[pred]
        space = max(len(word), len(pred))
        _text = "".join(([c for c in word] + [" "] * space)[:space])
        _tags = "".join(([c for c in pred] + [" "] * space)[:space])
        line_text += _text + " "
        line_tags += _tags + " "
    print(line_text)
    print(line_tags)

def main(args):
    # Build the config class
    config = Config(args, load=True)

    # Define the model
    model = RNN(config)

    # Build model
    model.load_model(config.dir)

    # Applications
    if not (args.text == None or args.tags == None):
        dataset = config.build_dataset(text_list=[args.text], tags_list=[args.tags])
        metrics = evaluate(model, dataset, config)
        print("Precision: ", metrics["precision"], "\nRecall:    ",
              metrics["recall"], "\nF1:        ", metrics["f1"])

    if not(args.input == None or args.output == None):
        # Check if files exist
        if not utils.check_file(args.input):
            raise Exception("File ", args.input, " does not exist")

        # Read file
        text = list(open(args.input, "r").readlines())

        # Get the predictions
        predictions = classify(text=text, model=model, config=config)

        # Make a dict for each prediction with the entities found
        for i, (pred, line) in enumerate(zip(predictions, text)):
            line = line.strip().split(" ")
            entities = {key: [] for key, value in config.tags.items()}
            for tag, word in zip(pred, line):
                tag = config.id_to_tag[tag]
                if not tag == "O" and not word in entities[tag]:
                    entities[tag].append(word)
            predictions[i] = entities

        # Output a file with [text, prediction]
        with open(args.output, "w") as w:
            for line, pred in zip(text, predictions):
                for key, value in pred.items():
                    pred[key] = " ".join(value)
                pred = pred["ORG"] + " " + pred["PRO"] + " " + \
                    pred["VER"] + " " + pred["VUL"] + " " + pred["ID"]
                w.write(line.strip()+"\t"+str(pred)+"\n")

if __name__ == '__main__':
    # Get arguments
    parser = argparse.ArgumentParser(description='Load a NER model')
    parser.add_argument('-model', metavar='Model', default=None,
                        help='(dir for the checkpoint folder)')
    parser.add_argument('-input', metavar='Input file', default=None,
                        help='(Unlabelled file)')
    parser.add_argument('-output', metavar='Ouput file', default=None,
                        help='(Output file)')
    parser.add_argument('-text', metavar='Positive File', default=None,
                        help='(Positive file)')
    parser.add_argument('-tags', metavar='Negative File', default=None,
                        help='(Negative file)')
    args = parser.parse_args()

    # Need to pass a dir where the model is loaded
    if args.model == None:
        raise Exception('Provide a model')

    main(args)


# -*- coding: utf-8 -*-
"""
Created on Mon Apr 26 14:44:18 2021

@author: sp3660
"""

import pandas as pd
import numpy as np
import sqlite3
from sqlite3 import Error

excelfile=r'C:\Users\sp3660\Desktop\ForMouseDatabase.xlsx'
databasefile=r'C:\Users\sp3660\Desktop\MouseDatabase.db'

#%%
Rooms_table = pd.read_excel(
    excelfile,
    sheet_name='Rooms',
    header=0)

Action_types_table = pd.read_excel(
    excelfile,
    sheet_name='Action Types',
    header=0)

Lines_table = pd.read_excel(
    excelfile,
    sheet_name='Lines',
    header=0)

Genes_table = pd.read_excel(
    excelfile,
    sheet_name='Genes',
    header=0)

Genotypes_table = pd.read_excel(
    excelfile,
    sheet_name='Genotypes',
    header=0)

Genotyping_status_table = pd.read_excel(
    excelfile,
    sheet_name='Genotypings',
    header=0)

Breeding_status_table = pd.read_excel(
    excelfile,
    sheet_name='Breeders',
    header=0)

Experimental_status_table = pd.read_excel(
    excelfile,
    sheet_name='Experimental',
    header=0)

Labels_table = pd.read_excel(
    excelfile,
    sheet_name='Labels',
    header=0)

Sex_table = pd.read_excel(
    excelfile,
    sheet_name='Sex',
    header=0)

Mice_table = pd.read_excel(
    excelfile,
    sheet_name='Mouse Table',
    header=0)

Mice_table.dtypes
Mice_table['Cage'] = Mice_table['Cage'].astype('Int64')
Mice_table.dtypes

Breedings_table = pd.read_excel(
    excelfile,
    sheet_name='Breedings',
    header=0)
Breedings_table.dtypes
Breedings_table['Cage'] = Breedings_table['Cage'].astype('Int64')
Breedings_table['Male'] = Breedings_table['Male'].astype('Int64')
Breedings_table['Female1'] = Breedings_table['Female1'].astype('Int64')
Breedings_table['Female2'] = Breedings_table['Female2'].astype('Int64')
Breedings_table['Male_Cage'] = Breedings_table['Male_Cage'].astype('Int64')
Breedings_table['Requires_Genotyping'] = Breedings_table['Requires_Genotyping'].astype('Int64')
Breedings_table.dtypes

Litters_table = pd.read_excel(
    excelfile,
    sheet_name='Litters',
    header=0)
Litters_table.dtypes
Litters_table['NumberDead'] = Litters_table['NumberDead'].astype('Int64')

Actions_table = pd.read_excel(
    excelfile,
    sheet_name='Actions',
    header=0)
Actions_table.dtypes
Actions_table['Mouse_2'] = Actions_table['Mouse_2'].astype('Int64')
Actions_table['Mouse_3'] = Actions_table['Mouse_3'].astype('Int64')
Actions_table['Mouse_4'] = Actions_table['Mouse_4'].astype('Int64')
Actions_table['Mouse_5'] = Actions_table['Mouse_5'].astype('Int64')
Actions_table.dtypes

#%%

db_conn = sqlite3.connect(databasefile)
c = db_conn.cursor()
#%%
c.execute(
    """
CREATE TABLE Sex_table (
    ID INTEGER PRIMARY KEY AUTOINCREMENT ,
    Sex_types TEXT
    );
"""
)

c.execute(
    """
CREATE TABLE Labels_table (
    ID INTEGER PRIMARY KEY AUTOINCREMENT ,
    Labels_types TEXT
    );
"""
)

c.execute(
    """
CREATE TABLE Experimental_table (
    ID INTEGER PRIMARY KEY AUTOINCREMENT ,
    Experimental_types TEXT
    );
"""
)

c.execute(
    """
CREATE TABLE Breeders_table (
    ID INTEGER PRIMARY KEY AUTOINCREMENT,
    Breeders_types TEXT
    );
"""
)

c.execute(
    """
CREATE TABLE Genotypings_table (
    ID INTEGER PRIMARY KEY AUTOINCREMENT ,
    Genotypings_types TEXT
    );
"""
)

c.execute(
    """
CREATE TABLE Genotypes_table (
    ID INTEGER PRIMARY KEY AUTOINCREMENT ,
    Genotypes_types TEXT
    );
"""
)

c.execute(
    """
CREATE TABLE Genes_table (
    ID INTEGER PRIMARY KEY AUTOINCREMENT ,
    Genes_types TEXT
    );
"""
)

c.execute(
    """
CREATE TABLE Lines_table (
    ID INTEGER PRIMARY KEY AUTOINCREMENT ,
    Line_Short TEXT,
    Line_Long TEXT
    );
"""
)

c.execute(
    """
CREATE TABLE Action_types_table (
    ID INTEGER PRIMARY KEY AUTOINCREMENT ,
    Action_types  TEXT
    );
"""
)

c.execute(
    """
CREATE TABLE Rooms_table (
    ID INTEGER PRIMARY KEY AUTOINCREMENT ,
    Room_types TEXT
    );
"""
)
#%%
c.execute(
    """
CREATE TABLE Breedings_table (
    ID INTEGER PRIMARY KEY AUTOINCREMENT ,
    Cage INTEGER,
    Male INTEGER,
    Female1 INTEGER,
    Female2 INTEGER,
    Line INTEGER NOT NULL,
    StartDate TEXT,
    EndDate TEXT,
    Male_Cage INTEGER,
    Breeding_Status INTEGER,
    Requires_Genotyping INTEGER,
    Notes TEXT,

    FOREIGN KEY (Line) REFERENCES Lines_table (ID)
    FOREIGN KEY (Male) REFERENCES MICE_table (ID)
    FOREIGN KEY (Female1) REFERENCES MICE_table (ID)
    FOREIGN KEY (Female2) REFERENCES MICE_table (ID)
    FOREIGN KEY (Breeding_Status) REFERENCES Breeders_table (ID)

    );
"""
)

c.execute(
    """
CREATE TABLE Litters_table (
    ID INTEGER PRIMARY KEY AUTOINCREMENT ,
    Cage INTEGER NOT NULL,
    Breeding_Parents INTEGER NOT NULL,
    Date_seen TEXT NOT NULL,
    Age INTEGER NOT NULL,
    Line INTEGER NOT NULL,
    Date_Weaned TEXT,
    NumberAlive INTEGER,
    NumberDead INTEGER,
    Notes TEXT,

    FOREIGN KEY (Line) REFERENCES Lines_table (ID)
    FOREIGN KEY (Breeding_Parents) REFERENCES Breedings_table(ID)

    );
"""
)

c.execute(
    """
CREATE TABLE Actions_table (
    ID INTEGER PRIMARY KEY AUTOINCREMENT,
    Date TEXT NOT NULL,
    Action_Type INTEGER NOT NULL,
    Cage_start INT,
    Cage_end INT,
    Mouse_1 INTEGER NOT NULL,
    Mouse_2 INTEGER,
    Mouse_3 INTEGER,
    Mouse_4 INTEGER,
    Mouse_5 INTEGER,
    Notes TEXT,

    FOREIGN KEY (Action_Type) REFERENCES Action_types_table (ID)
    FOREIGN KEY (Mouse_1) REFERENCES MICE_table(ID)
    FOREIGN KEY (Mouse_2) REFERENCES MICE_table(ID)
    FOREIGN KEY (Mouse_3) REFERENCES MICE_table(ID)
    FOREIGN KEY (Mouse_4) REFERENCES MICE_table(ID)
    FOREIGN KEY (Mouse_5) REFERENCES MICE_table(ID)

    );
"""
)

#%%
c.execute(
    """
CREATE TABLE MICE_table (
    ID INTEGER PRIMARY KEY AUTOINCREMENT ,
    Lab_Number  INTEGER NOT NULL,
    Sex INTEGER NOT NULL,
    Cage INTEGER,
    Breeding_status INTEGER NOT NULL,
    DOB TEXT,
    Label INTEGER NOT NULL,
    Parent_Breeding INTEGER NOT NULL,
    Ai14 INTEGER NOT NULL,
    Ai65 INTEGER NOT NULL,
    Ai75 INTEGER NOT NULL,
    Ai80 INTEGER NOT NULL,
    Ai148 INTEGER NOT NULL,
    Ai162 INTEGER NOT NULL,
    G2C INTEGER NOT NULL,
    VGC INTEGER NOT NULL,
    VRC INTEGER NOT NULL,
    PVF INTEGER NOT NULL,
    SLF INTEGER NOT NULL,
    Line INTEGER NOT NULL,
    Room INTEGER NOT NULL,
    Experimental_Status INTEGER NOT NULL,
    Genotyping_Status INTEGER NOT NULL,
    Alive INTEGER NOT NULL,
    Notes TEXT,

    FOREIGN KEY (Sex) REFERENCES Sex_table (ID)
    FOREIGN KEY (Breeding_status) REFERENCES Breeders_table (ID)
    FOREIGN KEY (Label) REFERENCES Labels_table (ID)
    FOREIGN KEY (Parent_Breeding) REFERENCES Breedings_table (ID)
    FOREIGN KEY (Ai14) REFERENCES Genotypes_table (ID)
    FOREIGN KEY (Ai65) REFERENCES Genotypes_table (ID)
    FOREIGN KEY (Ai75) REFERENCES Genotypes_table (ID)
    FOREIGN KEY (Ai80) REFERENCES Genotypes_table (ID)
    FOREIGN KEY (Ai148) REFERENCES Genotypes_table (ID)
    FOREIGN KEY (Ai162) REFERENCES Genotypes_table (ID)
    FOREIGN KEY (VGC) REFERENCES Genotypes_table (ID)
    FOREIGN KEY (VRC) REFERENCES Genotypes_table (ID)
    FOREIGN KEY (PVF) REFERENCES Genotypes_table (ID)
    FOREIGN KEY (SLF) REFERENCES Genotypes_table (ID)
    FOREIGN KEY (Line) REFERENCES Lines_table (ID)
    FOREIGN KEY (Room) REFERENCES Rooms_table (ID)
    FOREIGN KEY (Experimental_Status) REFERENCES Experimental_table (ID)
    FOREIGN KEY (Genotyping_Status) REFERENCES Genotypings_table (ID)

    );
"""
)
#%%

Sex_table.to_sql('Sex_table', db_conn, if_exists='append', index=False)
Rooms_table.to_sql('Rooms_table', db_conn, if_exists='append', index=False)
Action_types_table.to_sql('Action_types_table', db_conn, if_exists='append', index=False)
Lines_table.to_sql('Lines_table', db_conn, if_exists='append', index=False)
Genes_table.to_sql('Genes_table', db_conn, if_exists='append', index=False)
Genotypes_table.to_sql('Genotypes_table', db_conn, if_exists='append', index=False)
Genotyping_status_table.to_sql('Genotypings_table', db_conn, if_exists='append', index=False)
Breeding_status_table.to_sql('Breeders_table', db_conn, if_exists='append', index=False)
Experimental_status_table.to_sql('Experimental_table', db_conn, if_exists='append', index=False)
Labels_table.to_sql('Labels_table', db_conn, if_exists='append', index=False)
Breedings_table.to_sql('Breedings_table', db_conn, if_exists='append', index=False)
Litters_table.to_sql('Litters_table', db_conn, if_exists='append', index=False)
Actions_table.to_sql('Actions_table', db_conn, if_exists='append', index=False)
Mice_table.to_sql('MICE_table', db_conn, if_exists='append', index=False)

#%%
db_conn = sqlite3.connect(databasefile)
c = db_conn.cursor()
db_conn.close()

#%%

# query3="""
# SELECT
#     Lab_number,
#     Sex_types,
#     Cage,
#     round(julianday('now') - julianday(DOB)) AS DaysOld,
#     round(round(julianday('now') - julianday(DOB))/7) AS WeeksOld,
#     Breeders_types,
#     Labels_types,
#     Line_Short,
#     Genotypings_types

# FROM MICE_table

# LEFT JOIN Breeders_table ON Breeders_table.ID=MICE_table.Breeding_status
# LEFT JOIN Labels_table ON Labels_table.ID=MICE_table.Label
# LEFT JOIN Sex_table ON Sex_table.ID=MICE_table.Sex
# LEFT JOIN Experimental_table ON Experimental_table.ID=MICE_table.Experimental_Status
# LEFT JOIN Genotypings_table ON Genotypings_table.ID=MICE_table.Genotyping_Status
# LEFT JOIN Lines_table ON Lines_table.ID=MICE_table.Line
# LEFT JOIN Rooms_table ON Rooms_table.ID=MICE_table.Room

# WHERE Alive=1 AND (Experimental_Status=1 OR Experimental_Status=3)  AND Breeding_status=3

# """

# Alive_non_exp = pd.read_sql_query(query3,db_conn)

# Alive_non_exp.sort_values(by=['Sex_types'],ascending=False)
# new=    Alive_non_exp.sort_values(by=['Sex_types'],ascending=False)

# gk = new.groupby(['Breeders_types', 'Cage', 'Line_Short'])['Lab_Number'].apply(list)
# zz=gk.to_frame()
# tt=pd.DataFrame(zz.Lab_Number.values.tolist(),zz.index).add_prefix('Mouse_')
# %% changing the breeding mouse codes to indexes

# Breedings_table = pd.read_excel(
#     excelfile,
#     sheet_name='Breedings',
#     header=0)

# ChangeBreedings=Breedings_table
#%%

#             #%%
# unique=list(ChangeBreedings.Male.unique())
# list_to_replace=unique[:-2]
# list_to_replace.remove(np.nan)
# new_list=[Mice_table._get_value(Mice_table.index[Mice_table['Lab_Number'] == mouse].tolist()[0],  'ID') for  mouse in list_to_replace ]
# ChangeBreedings[['Male']]=ChangeBreedings[['Male']].replace(list_to_replace,new_list)

# unique=list(ChangeBreedings.Female1.unique())
# list_to_replace=unique[:-2]
# list_to_replace.remove(np.nan)
# new_list=[Mice_table._get_value(Mice_table.index[Mice_table['Lab_Number'] == mouse].tolist()[0],  'ID') for  mouse in list_to_replace ]
# ChangeBreedings[['Female1']]=ChangeBreedings[['Female1']].replace(list_to_replace,new_list)

# unique=list(ChangeBreedings.Female2.unique())
# list_to_replace=unique[:-2]
# list_to_replace.remove(np.nan)
# new_list=[Mice_table._get_value(Mice_table.index[Mice_table['Lab_Number'] == mouse].tolist()[0],  'ID') for  mouse in list_to_replace ]
# ChangeBreedings[['Female2']]=ChangeBreedings[['Female2']].replace(list_to_replace,new_list)

#%% change action lab number to codes

# Actions_table = pd.read_excel(
#     excelfile,
#     sheet_name='Actions',
#     header=0)

# unique=list(Actions_table.Mouse_1.unique())
# list_to_replace=unique
# new_list=[Mice_table._get_value(Mice_table.index[Mice_table['Lab_Number'] == mouse].tolist()[0],  'ID') for  mouse in list_to_replace ]
# Actions_table[['Mouse_1']]=Actions_table[['Mouse_1']].replace(list_to_replace,new_list)

# unique=list(Actions_table.Mouse_2.unique())
# list_to_replace=unique
# list_to_replace.pop(9)
# new_list=[Mice_table._get_value(Mice_table.index[Mice_table['Lab_Number'] == mouse].tolist()[0],  'ID') for  mouse in list_to_replace ]
# Actions_table[['Mouse_2']]=Actions_table[['Mouse_2']].replace(list_to_replace,new_list)

# unique=list(Actions_table.Mouse_3.unique())
# list_to_replace=unique
# list_to_replace.pop(3)
# new_list=[Mice_table._get_value(Mice_table.index[Mice_table['Lab_Number'] == mouse].tolist()[0],  'ID') for  mouse in list_to_replace ]
# Actions_table[['Mouse_3']]=Actions_table[['Mouse_3']].replace(list_to_replace,new_list)

# unique=list(Actions_table.Mouse_4.unique())
# list_to_replace=unique
# list_to_replace.pop(2)
# new_list=[Mice_table._get_value(Mice_table.index[Mice_table['Lab_Number'] == mouse].tolist()[0],  'ID') for  mouse in list_to_replace ]
# Actions_table[['Mouse_4']]=Actions_table[['Mouse_4']].replace(list_to_replace,new_list)

# unique=list(Actions_table.Mouse_5.unique())
# list_to_replace=unique
# list_to_replace.pop(0)
# new_list=[Mice_table._get_value(Mice_table.index[Mice_table['Lab_Number'] == mouse].tolist()[0],  'ID') for  mouse in list_to_replace ]
# Actions_table[['Mouse_5']]=Actions_table[['Mouse_5']].replace(list_to_replace,new_list)

# Actions_table['Mouse_2'] = Actions_table['Mouse_2'].astype('Int64')
# Actions_table['Mouse_3'] = Actions_table['Mouse_3'].astype('Int64')
# Actions_table['Mouse_4'] = Actions_table['Mouse_4'].astype('Int64')
# Actions_table['Mouse_5'] = Actions_table['Mouse_5'].astype('Int64')


from rabbitpy import Connection, Queue

import sys
import os

sys.path.append(
    os.path.dirname(
        os.path.abspath(
            os.path.dirname(os.path.abspath(os.path.dirname(__file__)))
        )
    )
)

from config.config_rabbitpy import URL

with Connection(URL) as conn:
    with conn.channel() as channel:
        queue = Queue(channel=channel, name="durable-test", durable=True)

        if queue.declare():
            print("Queue declared")


import requests
import telegram
from telegram.ext import CommandHandler, MessageHandler, Filters, Updater
import torch
from torchvision.datasets import ImageFolder
import torchvision.transforms as transforms
import os
import datetime
from torch.utils.data import DataLoader
import torch.nn as nn

# Define the image folder path
IMAGE_FOLDER = 'images'

# Create the image folder if it doesn't exist
if not os.path.exists(IMAGE_FOLDER):
    os.makedirs(IMAGE_FOLDER)

# Define the transformations to be applied to the images
transform = transforms.Compose([
    transforms.Resize((64, 64)),
    transforms.ToTensor(),
    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
])

# Create a dataset from the images in the directory
dataset = ImageFolder(root=IMAGE_FOLDER, transform=transform)

# Create a data loader that loads images from the dataset in batches
batch_size = 64
dataloader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, shuffle=True)

batch_size = 32
shuffle = True

data_loader = DataLoader(dataset, batch_size=batch_size, shuffle=shuffle)

# Save the data loader to a file so that it can be used in the training script
torch.save(data_loader, 'data_loader.pth')

# Load the trained generator network
generator = torch.load('data_loader.pth')

# Replace YOUR_TOKEN with your bot's API token
bot = telegram.Bot(token='put your bot api here')

def start(update, context, my_generator):
    context.bot.send_message(chat_id=update.effective_chat.id, text="Hello! Please describe the image you want me to generate. Be as specific as possible, such as the colors, objects, shapes, etc.")
    # if user sends a description, generate an image from it
    if not update.message.text:
        context.bot.send_message(chat_id=update.effective_chat.id, text="Please describe the image you want me to generate. Be as specific as possible, such as the colors, objects, shapes, etc.")
    else:
        generate_image(update, context, my_generator)

def train(update, context):
    # Check if there is a saved image for the user
    user_image_path = os.path.join(IMAGE_FOLDER, str(update.effective_chat.id) + '.jpg')
    if not os.path.exists(user_image_path):
        context.bot.send_message(chat_id=update.effective_chat.id, text="Please send me an image to train the model.")
        return
    # Call the save_image function to save the image
    save_image(update, context)

def save_image(update, context):
    # Checl if the message contains an image
    if not update.message.photo:
        context.bot.send_message(chat_id=update.effective_chat.id, text="Please send me an image.")
        return
    # Get the file ID of the image sent by the user
    file_id = update.message.photo[-1].file_id
    # Download the photo
    file = context.bot.get_file(file_id)
    # Name the image file with the user's ID
    now = datetime.datetime.utcnow().strftime("%Y-%m-%d-%H-%M-%S-UTC")
    filename = f"{update.effective_chat.id}-{now}.jpg"
    # Save the image file
    file.download(os.path.join(IMAGE_FOLDER, filename))
    context.bot.send_message(chat_id=update.effective_chat.id, text="Image saved successfully!")

def generate_image(update, context, my_generator):
    # Get the user's input from the message text
    input_text = update.message.text

    # Convert the user's input into a latent code
    latent_code = get_latent_code(input_text, my_generator)

    # Generate an image from the latent code
    with torch.no_grad():
        image = my_generator(latent_code)

    # Convert the image tensor to a PIL Image and save it
    transform = transforms.ToPILImage()
    image = transform(image)
    image.save('generated_image.jpg')

    # Send the generated image to the user
    with open('generated_image.jpg', 'rb') as f:
        context.bot.send_photo(chat_id=update.effective_chat.id, photo=f)

def echo(update, context):
    context.bot.send_message(chat_id=update.effective_chat.id, text=update.message.text)

def get_latent_code(input_text, my_generator):
    # Define the input size, hidden size, and latent size
    input_size = 100
    hidden_size = 256
    latent_size = 100

    # Define the text encoder network
    text_encoder = nn.Sequential(
        nn.Linear(input_size, hidden_size),
        nn.ReLU(),
        nn.Linear(hidden_size, latent_size),
        nn.Tanh()
    )
    # Convert the input text to a tensor
    text_tensor = torch.Tensor([input_text])

    # Generate a latent code from the text tensor
    with torch.no_grad():
        latent_code = text_encoder(text_tensor)

    return latent_code

# command handler to get image from description
def generate_images(update, context):
    description = update.message.text.split('/generate')[1]
    r = requests.get(f'https://api.unsplash.com/photos/random?query={description}&client_id=xKiAshqqBCZJ1QfaXMhkvCLfT_qhvcZq64kVfzMzoX8')
    if r.status_code == 200:
        photo_url = r.json()['urls']['regular']
        bot.send_photo(chat_id=update.effective_chat.id, photo=photo_url)
    else:
        bot.send_message(chat_id=update.effective_chat.id, text='Sorry, could not find an image.')

# message handler to reply to non-command messages
def reply(update, context):
    bot.send_message(chat_id=update.effective_chat.id, text='Sorry, I did not understand that. Please use /generate to get an image from a description.')

def main():
    updater = Updater(token='put your bot api here', use_context=True)
    dispatcher = updater.dispatcher

    # Create a generator object
    #generator = MyGenerator()
    # Define my_generator
    my_generator = generator

    # Register the start() function with the generator argument
    start_handler = CommandHandler('start', lambda update, context: start(update, context, my_generator), pass_args=True, pass_job_queue=True, pass_chat_data=True)

    #start_handler = CommandHandler('start', start)
    generate_image_handler = MessageHandler(Filters.text & (~Filters.command), generate_image)
    train_handler = CommandHandler('train', train)
    image_handler = MessageHandler(Filters.photo & (~Filters.command), save_image)
    echo_handler = MessageHandler(Filters.text & (~Filters.command), echo)

    # add handlers to dispatcher
    dispatcher.add_handler(CommandHandler('generate', generate_images))
    dispatcher.add_handler(MessageHandler(Filters.text & (~Filters.command), reply))
    dispatcher.add_handler(start_handler)
    dispatcher.add_handler(generate_image_handler)
    dispatcher.add_handler(train_handler)
    dispatcher.add_handler(image_handler)
    dispatcher.add_handler(echo_handler)

    updater.start_polling()
    updater.idle()

if __name__ == '__main__':
    main()


"""
Script solving the famous Travel Salesman Problem (aka TSP).

Script can be given the following parameters:
    * --nogui: disables graphic user interface display.
    * --maxtime N: specifies maximum time in seconds for the script to look for
        a solution.
    * --filename FILE: uses the file given as input to set the cities list.
    * --maxstagnation N: specifies the number of iterations with stagnation
        before stopping the algorithm.

Authors: M'Poy Julien & Perez Joaquim
Date: 15.01.2017
Location: Haute École Arc, Neuchâtel
"""
from math import ceil
from itertools import chain
import time
import copy
import random
import pygame
import sys
from operator import attrgetter
from functools import total_ordering
from pygame.locals import (
    KEYDOWN, QUIT, MOUSEBUTTONDOWN, K_RETURN, K_ESCAPE,
    K_SPACE
)

"""
City class
"""

class City:
    """Class representing a city."""

    def __init__(self, name, position):
        """Name is city's name and position is a two floats tuple."""
        self.name = name
        self.position = position

        # will contain the next city and the distance to it
        # when creating a solution
        self.next = None
        self.dist = None

    def __eq__(self, other):
        try:
            return self.name == other.name and self.position == other.position
        except AttributeError:
            return False

    def __repr__(self):
        return "{} {}\n".format(self.name, self.position)

    def __hash__(self):
        return hash(self.name)

    def get_closest(self, cities):
        closest = cities[0]
        closest_dist = self.distance_to(closest)
        for city in cities:
            if self.distance_to(city) < closest_dist:
                closest = city
        return closest

    def distance_to(self, other):
        dx, dy = (
            self.position[0] - other.position[0],
            self.position[1] - other.position[1]
        )
        return (dx*dx + dy*dy) ** .5

"""
Solution
"""

@total_ordering
class Solution:
    def __init__(self, cities):
        """
        cities is an ordered list of cities, the last one is linked to the
        first one
        """
        self.cities = copy.copy(cities)
        self.compute_fitness()

    def compute_fitness(self):
        self.fitness = 0
        self.dict = {}

        old_city = self.cities[0]
        for city in chain(self.cities + [old_city]):
            distance = old_city.distance_to(city)
            # add the distance between this city and
            # the previous one to the fitness
            self.fitness += distance
            old_city.next = city
            old_city.dist = distance
            self.dict[old_city.name] = old_city
            old_city = city

    # total_ordering makes solutions comparable from these 2 methods
    def __eq__(self, other):
        return self.fitness == other.fitness

    def __lt__(self, other):
        return self.fitness < other.fitness

    def __repr__(self):
        return "solution with fitness {}".format(self.fitness)

    def mutate(self):
        """
        two blocks inversion (not indispensable)
        ABCDEF becomes CDEFAB

        The solution keeps the same, it only gives more chances to the FA
        segment to be muted.
        """
        pivot = random.randrange(len(self.cities))
        self.cities = self.cities[pivot:] + self.cities[:pivot]

        # i <- minimum born, j <- maximum born
        i = random.randrange(len(self.cities)-1)
        j = random.randrange(i+1, len(self.cities))

        # C
        # D
        # E <-- i
        # F
        # A <-- j
        # B

        top = self.cities[:i]  # CD
        middle = self.cities[i:j]  # EF
        bottom = self.cities[j:]  # AB
        # tidying block
        self.cities = middle + top + bottom  # EFCDAB

        self.compute_fitness()

    @staticmethod
    def create_pseudo_best(cities):
        # petite copie oklm pour ne pas toucher aux objets originaux
        cities = list(cities)
        ordered_cities = []
        city = cities[0]
        ordered_cities.append(city)
        cities.remove(city)
        while cities:
            city = city.get_closest(cities)
            ordered_cities.append(city)
            cities.remove(city)
        return Solution(ordered_cities)

"""
GUI
"""

class Gui:
    """Gui screen using pygame."""
    def __init__(self, cities=None, file_name=''):
        self.screen_x = 500
        self.screen_y = 500
        self.city_color = [10, 10, 200]  # blue
        self.city_radius = 3
        self.font_color = [255, 255, 255]  # white
        pygame.init()
        self.window = pygame.display.set_mode((self.screen_x, self.screen_y))
        pygame.display.set_caption('Travelling Salesman Problem')
        self.screen = pygame.display.get_surface()
        self.font = pygame.font.Font(None, 30)

        if cities:
            self.cities = cities
            self.draw_cities()
            self.text(file_name)
            pygame.display.flip()
            self.wait_for_user_input()
        else:
            self.cities = []
            self.place_cities()

    def wait_for_user_input(self):
        while True:
            event = pygame.event.wait()
            if (event.type == QUIT or (
                    event.type == KEYDOWN and event.key == K_ESCAPE
            )):
                sys.exit(0)
            elif (event.type == KEYDOWN and (
                    event.key == K_RETURN or event.key == K_SPACE
                )
            ):
                break

    def place_cities(self):
        city_counter = 0
        self.screen.fill(0)
        self.text("Placez vos point puis appuyez sur Enter")
        pygame.display.flip()
        while True:
            for event in pygame.event.get():
                if event.type == MOUSEBUTTONDOWN:
                    pos = pygame.mouse.get_pos()
                    name = "v{}".format(city_counter)
                    self.cities.append(City(name, pos))
                    city_counter += + 1
                    self.screen.fill(0)
                    self.draw_cities()
                    self.text("Nombre: {}".format(len(self.cities)))
                    pygame.display.flip()
                elif (event.type == QUIT or (
                        event.type == KEYDOWN and event.key == K_ESCAPE
                    )
                ):
                    sys.exit(0)
                elif (event.type == KEYDOWN and (
                    event.key == K_RETURN or event.key == K_SPACE
                    )
                ):
                    if(len(self.cities) > 2):
                        return

    def draw_cities(self):
        for city in self.cities:
            pygame.draw.circle(
                self.screen, self.city_color, city.position, self.city_radius
            )

    def draw_path(self, solution, msg="", color=[255, 0, 0]):
        self.screen.fill(0)
        pygame.draw.lines(self.screen, color, True, [
            city.position for city in solution.cities]
        )
        self.draw_cities()
        self.text(msg)
        pygame.display.flip()

    def text(self, msg, render=False):
        text = self.font.render(msg, True, self.font_color)
        textRect = text.get_rect()
        self.screen.blit(text, textRect)
        if render:
            pygame.display.flip()

"""
Algorithme
"""

def crossover_from_best_in_parents(father, mother):
    """
    Starting from a city randomly selected, we check both parents'
    nex city and choose the closest one. If this city is already present
    in the solution, we take the other parent's one. If this city is
    already in the solution too, we choose an other city randomly.
    """
    children = []
    for i in range(2):
        cities = []
        names = [city.name for city in father.cities]
        chosen_name = random.choice(names)
        while len(names) > 0:
            names.remove(chosen_name)
            current_city = father.dict[chosen_name]
            cities.append(current_city)
            from_father = father.dict[chosen_name]
            from_mother = mother.dict[chosen_name]

            candidates = []
            if from_father.next.name in names:
                candidates += [from_father]
            if from_mother.next.name in names:
                candidates += [from_mother]

            if len(candidates) > 0:
                # we can use a gene from one the parents
                candidates.sort(key=attrgetter('dist'))
                chosen_name = candidates[0].next.name
            else:
                """
                we cannot because both cities are already in the solution
                we are building.
                """
                if names:
                    chosen_name = random.choice(names)
        children += [Solution(cities)]

    return children

def ox_crossover(x, y, crossover_ratio=0.3):
    """
    OX Crossover between x and y indviduals. Crossover section is defined
    randomly following the crossover_ration parameter.

    The two children are generated and returned as a tuple containing.
    """
    x, y = x.cities, y.cities
    genes_to_crossover = int(ceil(len(x) * crossover_ratio))
    start = random.randint(0, len(x) - genes_to_crossover - 1)
    stop = start + genes_to_crossover
    x_crossover = tuple(x[start:stop])
    y_crossover = tuple(y[start:stop])
    prepared_x = [item if item not in y_crossover else None for item in x]
    x_shifts = prepared_x[stop:].count(None)
    prepared_y = [item if item not in x_crossover else None for item in y]
    y_shifts = prepared_y[stop:].count(None)
    new_x = [item for item in prepared_x if item is not None]
    new_y = [item for item in prepared_y if item is not None]
    new_x = shift_list(new_x, x_shifts)
    new_y = shift_list(new_y, y_shifts)
    new_x = new_x[:start] + list(y_crossover) + new_x[start:]
    new_y = new_y[:start] + list(x_crossover) + new_y[start:]
    return Solution(new_x), Solution(new_y)

def shift_list(items, shifts):
    """Left shift a list of elements."""
    for i in range(shifts):
        items = items[1:] + items[:1]
    return items

def ga_solve(file=None, gui=True, maxtime=0, maxstagnation=200):
    """
    Main function parsing file, initializing UI and launching genetic
    algorithm solving method.
    """

    # dictionnary containing the cities
    # key: city's name, value: City object
    cities = []

    # just because we are already using a gui variable
    gui_diplay = gui

    if file:
        with open(file, encoding='utf-8') as positions_file:
            for line in positions_file:
                # for each line, we create a City object with the coordinates
                cityname, x, y = line.split()
                cities.append(City(cityname, (int(x), int(y))))
            if gui_diplay:
                gui = Gui(cities, file)
    else:
        gui = Gui()
        cities = gui.cities

    t1 = time.time()

    POPULATION_SIZE = 20
    # int because division always returns a float
    HALF = int(POPULATION_SIZE/2)
    QUARTER = int(POPULATION_SIZE/4)
    # mutation rate
    rate = 0.2

    """
    RANGS contains n*0,  (n-1)*1 ... 1*n
        -> example 10, 9,9, 8,8,8, 7,7,7,7, ...
    """
    RANGS = [
        POPULATION_SIZE-i for i in range(POPULATION_SIZE+1) for j in range(i)
    ]

    population = []
    for i in range(POPULATION_SIZE):
        random.shuffle(cities)
        population.append(Solution(cities))

    pseudo_best = Solution.create_pseudo_best(cities)
    population.append(pseudo_best)

    old_best = 0
    stagnation = 0

    method_1 = True

    # genetic algorithm
    while True:
        # population sorting
        population.sort()

        best = Solution(population[0].cities)

        # population = population[:HALF] # selects half of the best solutions

        # more natural selection
        # select a quarter of the best solutions
        population = population[:QUARTER]
        # the select an other quarter randomly
        # population[QUARTER:] remains
        population += random.sample(population, QUARTER)

        # seems more correct but works gives worse results
        # elite, reste = population[:QUARTER], population[QUARTER:]
        # population = elite + random.sample(population, QUARTER)

        # population shuffling
        random.shuffle(population)

        # crossover
        for i in range(0, HALF, 2):  # 0 to HALF 2-by-2
            sol1 = population[i]
            sol2 = population[i+1]
            # add children to the population
            if method_1:
                population += crossover_from_best_in_parents(sol1, sol2)
            else:
                population += list(ox_crossover(sol1, sol2))
            method_1 = not method_1

        # muate 20% of the solutions
        [
            solution.mutate() for solution in random.sample(
                population, int(rate * len(population))
            )
        ]
        population.append(best)

        if best.fitness == old_best:
            stagnation += 1
            # stop if the best solution is the same n time consequently
            if not maxtime and stagnation == maxstagnation:
                break
        else:
            stagnation = 0
            if gui_diplay:
                gui.draw_path(best, msg=str(best.fitness))

        if maxtime:
            dt = time.time() - t1
            if dt > maxtime:
                break

        old_best = best.fitness

    if gui_diplay:
        gui.draw_path(best, msg=str(best.fitness), color=[0, 255, 0])
        gui.wait_for_user_input()

    path = [city.name for city in best.cities]
    return best.fitness, path

"""
MAIN
"""
if __name__ == "__main__":
    import argparse

    parser = argparse.ArgumentParser()
    parser.add_argument("-f", "--filename", help="use the file given as input")
    parser.add_argument(
        "-n", "--nogui",
        help="disable graphic user interface display", action="store_true"
    )
    parser.add_argument(
        "-t", "--maxtime", help="specify the maximum time in seconds", type=int
    )
    parser.add_argument(
        "-s", "--maxstagnation", type=int, default=200,
        help="specify the number of iteration with \
            stagnation before stopping the algorithm"
    )
    args = parser.parse_args()

    distance, path = ga_solve(
        file=args.filename,
        gui=not args.nogui,
        maxtime=args.maxtime,
        maxstagnation=args.maxstagnation
    )

    print("Distance : {}\nPath : {}".format(distance, path))


from __future__ import annotations
import io
import datetime

import asyncpg
import yarl
import re
import pprint
import tabulate
from typing import Optional, Union, cast, TYPE_CHECKING

import aiohttp
import discord

from discord.ext import commands

from utils import paginator, errors
from utils.context import Context

if TYPE_CHECKING:
    from bot import Bot

URL_RE = re.compile(r"http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*(),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+")

SPECIAL_CHANNEL_INDEX = {
    669115391880069150: "twitchio",
    491048464831086592: "twitchio",
    531269424523771934: "wavelink",
    739788459006492752: "wavelink",
    491048383578898441: "discord.py"
}

class _libconverter(commands.Converter):
    async def convert(self, ctx: Context, param):
        if param.lower() in {"dpy", "discordpy", "discord.py"}:
            return "discord.py-2"
        elif param.lower() in {"dpy2", "discord.py2", "discordpy2", "2.0", "master"}:
            return "discord.py-2"
        elif param.lower() in {"tio", "twitch", "twitchio"}:
            return "twitchio"
        elif param.lower() in {"wl", "wave", "link", "wavelink"}:
            return "wavelink"
        elif param.lower() in {"ahttp", "aiohttp"}:
            return "aiohttp"
        elif param.lower() in {"enhanced-dpy", "enhanced-discord.py", "edpy"}:
            return "enhanced-discord.py"
        else:
            raise commands.UserInputError("Must be one of discord.py, discord.py2, enhanced-dpy, twitchio, wavelink, or aiohttp")

async def setup(bot):
    await bot.add_cog(Idevision(bot))

v = {
    "link1": "https://github.com/",
    "link1_name": "Github",
    "link2": "https://metrics.idevision.net",
    "link2_name": "Idevision",
    "link3": "https://twitch.tv",
    "link3_name": "Twitch",
    "link4": "https://youtube.com",
    "link4_name": "Youtube"
}

class Idevision(commands.Cog):
    url = "https://idevision.net/api/"

    def __init__(self, bot: Bot):
        self.bot = bot
        self.defaults = {}
        self.pages = {}
        self.usage = {}
        self.session = aiohttp.ClientSession(headers={"Authorization": bot.config['idevision']['token'], "User-Agent": "MiniBOB"})
        self._hook_session = aiohttp.ClientSession()
        self.hook: discord.Webhook = discord.Webhook.from_url(bot.config['idevision']['webhook_url'], session=self._hook_session) # noqa

    async def cog_load(self) -> None:
        self.db: asyncpg.Pool = await asyncpg.create_pool(self.bot.config["db"]["dsn"])

    async def cog_unload(self):
        await self._unload()

    async def _unload(self):
        await self.db.close()
        await self.session.close()
        await self._hook_session.close()

    async def do_rtfm(self, ctx, key, obj, labels=True, obvious_labels=False):
        if ctx.guild is not None and ctx.guild.id in self.defaults and key is None:
            key = self.defaults[ctx.guild.id]

        target = self.pages[key]['url']

        if obj is None:
            return await ctx.send(target)

        url = yarl.URL(self.url + "public/rtfm.sphinx").with_query(
            {
                "query": obj,
                "location": target,
                "show-labels": str(labels),
                "label-labels": str(obvious_labels)
            }
        )

        headers = {"User-Agent": f"BOB discord bot (squawking {ctx.author})"}
        async with self.session.get(url, headers=headers) as resp:
            if resp.status != 200:
                return await ctx.send(f"The api returned an irregular status ({resp.status}) ({await resp.text()})")

            matches = await resp.json()
            if not matches['nodes']:
                return await ctx.send("Could not find anything. Sorry.")

        e = discord.Embed(colour=0x36393E)
        e.title = f"{self.pages[key]['long']}: {obj}"
        e.description = '\n'.join(f'[`{key}`]({url})' for key, url in matches['nodes'].items())
        e.url = self.url.replace("api/", "docs")
        e.set_footer(text=f"rtfm api available at {e.url}")
        await ctx.send(embed=e)

    @commands.group(aliases=['rtfd', "rtm", "rtd"], invoke_without_command=True, help="read the fucking docs. see `!help rtfm`")
    async def rtfm(self, ctx: Context, *, obj: Optional[str] = None):
        """
        view the documentation of the modules available in `rtfm list`.
        use their *quick* name to access it in the rtfm command, as such:
        `rtfm py sys`

        By default, labels are not shown in the results. To enable labels, use the `--labels` flag
        """
        await ctx.typing()

        labels = False
        obvious_labels = False
        if obj is not None:
            if "--labels" in obj:
                labels = True
                obvious_labels = True
                obj = obj.replace("--labels", "")

            from discord.ext.commands.view import StringView
            view = StringView(obj)
            key = view.get_word()  # check if the first arg is specifying a certain rtfm
            approved_key = None

            for k, v in self.pages.items():
                if key == k:
                    approved_key = k
                    view.skip_ws()
                    obj = view.read_rest().strip()
                    if not obj: obj = None
                    break

                elif key.lower() == v['long'].lower():
                    approved_key = k
                    view.skip_ws()
                    obj = view.read_rest().strip()
                    if not obj:
                        obj = None
                    break

            if approved_key is None:
                obj = (key + " " + view.read_rest()).strip()
                if not obj: obj = None

            if not approved_key and ctx.channel.id in SPECIAL_CHANNEL_INDEX:
                approved_key = list(filter(lambda m: m['long'].lower() == SPECIAL_CHANNEL_INDEX[ctx.channel.id],
                                           self.pages.values()))[0]['quick']

            if not approved_key and ctx.guild.id in self.defaults:
                approved_key = self.defaults[ctx.guild.id]

            elif not approved_key:
                raise errors.CommandInterrupt("No rtfm selected, and no default doc is set for your guild.")

        elif ctx.guild.id in self.defaults:
            approved_key = self.defaults[ctx.guild.id]
        else:
            raise errors.CommandInterrupt("No rtfm selected, and no default doc is set for your guild.")

        self.usage[approved_key] = datetime.datetime.utcnow()
        await self.do_rtfm(ctx, approved_key, obj, labels, obvious_labels)

    @rtfm.command()
    async def list(self, ctx):
        """
        shows a list of the current documentation entries. you can use the short name to use the doc. ex: !rtfm py {{insert thing here}}
        """
        all_entries = await self.db.fetch("SELECT * FROM pages")
        entries = [(a['quick'], f"{a['long']}: {a['url']}") for a in all_entries]
        pages = paginator.FieldPages(ctx, entries=entries, per_page=5)
        await pages.paginate()

    @rtfm.command()
    @commands.has_permissions(manage_guild=True)
    async def default(self, ctx: Context, default: str):
        """
        sets a default rtfm for your guild, so you don't need to type the docs prefix each time.
        requires the `Bot Editor` role or higher
        note that you can only have 1 default per guild.
        """
        if default not in self.pages:
            return await ctx.send(f"`{default}` is not a valid doc! If you wish to add one, please use `!rtfm add` to submit it for review")
        else:
            self.defaults[ctx.guild.id] = default
            await self.db.execute("INSERT INTO default_rtfm VALUES ($1,$2)", ctx.guild.id, default)
            await ctx.send(f"set the guild's default rtfm to `{self.pages[default]['long']}`")

    @rtfm.command(usage="")
    async def add(self, ctx: Context, quick=None, long=None, url=None):
        """
        Have some documentation you want to see here? Add it here!
        there are a few requirements for your docs to be approved.
        - it **must** be created with Sphinx.

        You will be prompted for the documentation information
        """
        if await ctx.bot.is_owner(ctx.author) and quick and long and url:
            if quick in self.pages:
                return await ctx.send("Already exists")
            await self.db.execute("INSERT INTO pages VALUES ($1,$2,$3)", quick, long, url)

            self.pages[quick] = {"quick": quick, "long": long, "url": url}
            return await ctx.send("\U0001f44d")

        def check_cancel(content):
            if "cancel" in content:
                raise errors.CommandInterrupt("aborting")

        await ctx.send("By adding documentation, you agree that you have read the rules to adding documentation. type cancel to abort the creation process\n\n"
                       "Please provide a quick default for your rtfm (used when not accessing your guild's default, max 7 characters)")
        msg = await ctx.bot.wait_for("message", check=lambda m: m.channel == ctx.channel and m.author == ctx.author, timeout=30)
        quick = await commands.clean_content().convert(ctx, msg.content)
        if len(quick) > 7:
            raise commands.CommandError("That's more than 7 characters!")

        check_cancel(msg.content)
        if quick in self.pages:
            raise commands.CommandError("That documentation already exists!")

        try:
            long = await commands.clean_content().convert(ctx, await ctx.ask("Now, please provide the full documentation name", return_bool=False))
            check_cancel(long)
        except errors.CommandInterrupt: raise
        except: pass

        msg = await ctx.ask("Now, provide the url to the documentation.", return_bool=False)
        check_cancel(msg)

        url = await commands.clean_content().convert(ctx, msg)
        async with self.bot.session.get(url.strip("/")+"/objects.inv") as resp:
            if resp.status != 200:
                raise errors.CommandInterrupt("Invalid url provided (no /objects.inv found). remember to remove the current page! ex. https://docs.readthedocs.io/latest")

        await self.db.execute("INSERT INTO pages VALUES ($1,$2,$3)", quick, long, url)
        self.pages[quick] = {"quick": quick, "long": long, "url": url}
        await ctx.send("Your documentation has been added")

    @rtfm.command(hidden=True)
    @commands.is_owner()
    async def remove(self, ctx: Context, quick: str):
        await self.db.execute("DELETE FROM pages WHERE quick = $1", quick)
        if quick in self.pages:
            del self.pages[quick]
            return await ctx.send(f"removed `{quick}` from rtfm")
        await ctx.send(f"`{quick}` not found")

    @rtfm.before_invoke
    @default.before_invoke
    @add.before_invoke
    async def rtfm_pre(self, ctx: Context):
        if not self.pages:
            data = await self.db.fetch("SELECT * FROM pages")
            for record in data:
                self.pages[record['quick']] = dict(record)

        if not self.defaults:
            v = await self.db.fetch("SELECT * FROM default_rtfm")
            for record in v:
                self.defaults[record['guild_id']] = record['name']

    @commands.command()
    async def xkcd(self, ctx: Context, *, no_or_search: Union[int, str]):
        nsfw = ctx.guild is None or cast(discord.TextChannel, ctx.channel).is_nsfw()

        if isinstance(no_or_search, int):
            async with self.bot.session.get(f"https://xkcd.com{no_or_search}/info.0.json") as resp:
                if resp.status == 404:
                    return await ctx.send("XKCD not found")

                resp = await resp.json()
                e = discord.Embed(title=resp['safe_title' if nsfw else "title"],
                                   description=resp['alt'], color=0x36393E)
                e.set_footer(text=f"#{resp['num']}  • {resp['month']}/{resp['day']}/{resp['year']}")
                e.set_image(url=resp['img'])
                await ctx.send(embed=e)

        else:
            async with self.session.get(self.url + f"public/xkcd?search={no_or_search}") as resp:
                if resp.status != 200:
                    return await ctx.send(f"Something messed up :( ({resp.status, resp.reason})")

                data = await resp.json()

            if not data['nodes']:
                return await ctx.send("Nothing found :(")

            ems = []
            for d in data['nodes']:
                e = discord.Embed(
                    title=d['safe_title' if nsfw else "title"],
                    description=d['alt'],
                    color=0x36393E,
                    url=d['url'],
                    timestamp=datetime.datetime.fromisoformat(d['posted'])
                ).set_footer(
                    text=f"#{d['num']}"
                ).set_image(
                    url=d['image_url']
                )
                ems.append(e)

            if len(ems) == 1:
                return await ctx.send(embed=ems[0])

            e = paginator.EmbedPages(ctx, entries=ems)
            await e.paginate()

    @commands.command("chess")
    @commands.is_owner()
    async def chess(self, ctx: Context, target: discord.User, board_theme: str="walnut"):
        th1 = await ctx.ask(f"{ctx.author.mention}, theme?", return_bool=False)
        th2 = await ctx.ask(f"{target.mention}, theme?", return_bool=False, target=target)

        async with self.session.post(self.url + "games/chess", json={"white-theme": th1, "black-theme": th2, "board-theme": board_theme}) as resp:
            if resp.status != 200:
                return await ctx.send(f"Failed to generate a game: {resp.reason}")

            board: dict = await resp.json()

        #board = {'pieces': ['1a11', '0a21', None, None, None, '0a60', None, '1a80', None, '0b21', None, None, None, '0b60', None, '2b80', None, '0c21', '2c31', None, None, '0c60', None, '3c80', None, '4d21', None, '0d41', None, '0d60', None, '4d80', '5e11', '0e21', '3e31', None, None, None, '0e70', '5e80', '3f11', '0f21', None, None, None, None, '0f70', '3f80', '2g11', '0g21', None, None, None, None, '0g70', '2g80', '1h11', '0h21', None, None, None, None, '0h70', '1h80'], 'turn': 1, 'transcript': ['10d2d4nn', '00d7d6nn', '13c1e3nn', '00b7b6nn', '12b1c3nn', '00c7c6nn', '14d1d2nn', '00a7a6nn'], 'white-theme': 'glass', 'black-theme': 'glass', 'board-theme': 'walnut', 'castling': '1111'}

        last: Optional[discord.Message] = None
        _last: Optional[discord.Message] = None
        arrow = None
        do_render = True

        while True:
            if last:
                await last.delete()
                if _last:
                    await _last.delete()
                last = _last = None

            if do_render:
                _last = await ctx.send(str((board, arrow)))
                async with self.session.post(self.url + "games/chess/render", json={"board": board, "arrow": arrow}) as resp:
                    if resp.status != 200:
                        return await ctx.send(f"Failed to generate a board: {resp.reason}")

                    data = io.BytesIO()
                    data.write(await resp.read())
                    data.seek(0)

                last = await ctx.send(file=discord.File(data, "board.png"))

            turn = board['turn']
            t: discord.Member | discord.User
            if turn == 1:
                t = ctx.author
            else:
                t = target

            while True:
                move = await ctx.ask(None, target=t, return_bool=False, timeout=None)
                if move in ("cancel", "stop"):
                    await ctx.send("\U0001f44d")
                    return

                if "-" in move:
                    break

                if move == "transcript":
                    async with self.session.post(self.url + "games/chess/transcript", json={"board": board}) as resp:
                        transcript = await resp.text()

                    if len(transcript) > 1500:

                        pages = paginator.TextPages(ctx, "Click \U000023f9 to continue with the game\n" + transcript, max_size=1500, prefix="", suffix="")
                        await pages.paginate()
                    else:
                        await ctx.send(transcript)

            async with self.session.post(self.url + "games/chess/turn", json={"board": board, "move": move, "move-turn": "black" if turn else "white"}) as resp:
                if resp.status == 417:
                    data = await resp.json()
                    board = data['board']
                    await ctx.send(f"expectation failed: {data['error']}")
                    do_render = True

                elif resp.status == 200:
                    data = await resp.json()
                    board = data['board']
                    arrow = data.get("arrow")
                    do_render = True

                else:
                    await ctx.send(f"something fucked up: {resp.status, resp.reason, await resp.text()}")

    @commands.command("math")
    async def math(self, ctx: Context, *, expression: str):
        r"""
        This math parser makes use of the idevision math endpoint.
        This accepts mathematical functions (P(x) = expression)
        \> Functions in the form of `y = ...` will be graphed, and will not be usable in expressions. These functions have one implicit argument, `x`.
        Geometric sequences can be created using the `s` variable: `s=1,2,4`. The third arg is optional. You can then access the sequence via `s(n)`
        The following variables are currently built in:
        - pi
        - E
        The following functions are currently built in:
        - sin(x) / asin(x, y)
        - cos(x) / acos(x, y)
        - tan(x) / atan(x, y)
        - log(n, base)
        Each line will be treated as a separate expression. Functions defined are available for all expressions.
        If you find something that returns a 500 error, or something is missing that'd you'd like to see in this endpoint, feel free to contact me @IAmTomahawkx

        *This endpoint is in heavy beta, and is inherently unstable*
        """
        headers = {"User-Agent": f"BOB discord bot (squawking {ctx.author})"}

        async with self.session.post(self.url + "public/math", data=expression, headers=headers) as resp:
            if resp.status == 417:
                await ctx.send(f"Your expression contains errors\n```\n{(await resp.text()).strip()}\n```",
                               allowed_mentions=discord.AllowedMentions.none())

            elif resp.status == 200:
                data = await resp.json()
                files = []
                if data['images']:
                    for i, x in enumerate(data['images']):
                        async with self.session.get(x) as r:
                            if r.status == 200:
                                files.append(discord.File(io.BytesIO(await r.read()), filename=f"file-{i}.png"))

                await ctx.send(data['text'], files=files)

            else:
                await ctx.send(f"The api returned a non-ok response ({resp.status}: {await resp.text()}). "
                               f"<@547861735391100931> something fucked up", allowed_mentions=discord.AllowedMentions.all())

    @commands.command("rtfm-rs")
    async def rtfm_rs(self, ctx: Context, crate: str, *, query: str):
        """
        Searches https://docs.rs/ crates using the idevision.net api. alternatively, pass `std` as the crate to search the stdlib docs.
        """
        if not crate.startswith("https") and crate != "std":
            crate = f"https://docs.rs/{crate}"

        headers = {"User-Agent": f"BOB discord bot (squawking {ctx.author})"}
        async with self.session.get(self.url + "public/rtfm.rustdoc", params={"location": crate, "query": query}, headers=headers) as resp:
            if resp.status != 200:
                return await ctx.send(f"Api returned a bad response {resp.status, resp.reason}")

            data = await resp.json()
            await ctx.send(data)

    @commands.command()
    async def rtfs(self, ctx: Context, lib: Optional[_libconverter], *, item: str): # type: ignore
        """
        Indexes a library for items matching the given input.
        The following libraries are valid: dpy, dpy2, aiohttp, twitchio, wavelink

        you can use the --source flag to receive the source directly in discord, instead of a link to github
        """
        lib: str = lib # type: ignore

        fmt: str = "links"
        if "--source" in item:
            item = item.replace("--source", "")
            fmt = "source"

        if lib is None:
            lib = SPECIAL_CHANNEL_INDEX.get(ctx.channel.id, "discord.py-2")

        url = yarl.URL(self.url + "public/rtfs").with_query(
            {
                "query": item,
                "library": cast(str, lib),
                "format": fmt
            }
        )
        headers = {"User-Agent": f"BOB discord bot (squawking {ctx.author})"}
        async with self.session.get(url, headers=headers) as resp:
            if resp.status != 200:
                return await ctx.reply(f"The api returned a non-ok response: {resp.status} ({resp.reason})", mention_author=False)

            data = await resp.json()

        nodes = data['nodes']
        time = float(data['query_time'])

        if not nodes:
            return await ctx.reply(f"Could not find anything. Sorry. (query time {round(time, 4)})", mention_author=False)

        if fmt == "links":
            out = [f"[{name}]({url})" for name, url in nodes.items()]

            await ctx.send(embed=discord.Embed(
                description="\n".join(out),
                title=f"Result for {lib}: {item}"
            )
                .set_footer(text=f"Is the api behind on commits? Use {discord.utils.escape_mentions(cast(str, ctx.prefix))}rtfs-reload")
                .set_author(name=f"query time: {round(time, 3)} • commit {data['commit'][:6]}"))

        else:
            n = next(iter(nodes.items()))
            await ctx.reply(f"Showing source for {n[0]}\nCommit: {data['commit']}", mention_author=False)
            pages = paginator.TextPages(ctx, n[1], prefix="```py")
            await pages.paginate()

    @commands.command("rtfs-reload")
    @commands.cooldown(1, 180)
    async def rtfs_reload(self, ctx):
        """
        Reloads all rtfs nodes up to the latest commit.
        This command should not be called frequently.
        """
        url = yarl.URL(self.url + "public/rtfs.reload")

        async with self.session.put(url) as resp:
            if resp.status != 200:
                return await ctx.send(f"The api returned a non-ok response: {resp.status} ({resp.reason})")

            data = await resp.json()

        if data['fail']:
            fails = "(Failed to reload: " + ", ".join(data['fails']) + ")"
        else:
            fails = ""

        commits = "\n".join(f"{name}: {commit}" for name, commit in data['commits'].items())

        await ctx.reply(f"Reloaded {len(data['success'])} repos {fails}.\n{commits}", mention_author=False)

    @commands.command()
    @commands.cooldown(4, 10)
    async def ocr(self, ctx: Context, attachment: Optional[str] = None):
        """
        Preforms OCR on an image. This image can be uploaded as an attachment, or passed as a url link.
        OCR api: https://idevision.net/docs
        """
        if not ctx.message.attachments and not attachment:
            return await ctx.send("Please provide an attachment or a url")

        if ctx.message.attachments:
            url = ctx.message.attachments[0].url
        else:
            url = attachment

        resp = await self.bot.session.get(url) #type: aiohttp.ClientResponse
        if "image" not in resp.content_type:
            resp.close()
            return await ctx.send("URL/attachment was not an image")

        filetype = (ctx.message.attachments[0].filename if ctx.message.attachments else cast(str, attachment).split("/")[-1]).split(".")[-1]

        headers = {"User-Agent": f"BOB discord bot (squawking {ctx.author})"}
        async with self.session.get(self.url + f"public/ocr?filetype={filetype}", data=resp.content, headers=headers) as r:
            resp.close()
            if r.status != 200:
                return await ctx.send(f"The api responded with {r.status}: {r.reason}")
            else:
                d = await r.json()
                pages = paginator.TextPages(ctx, d['data'])

    @commands.group(aliases=["id"], invoke_without_command=True)
    async def idevision(self, ctx):
        """
        API control for the idevision site.
        Most of this is owner-only, the `apply` and `token` subcommands are available for public use.
        """
        await ctx.send("Idevision documentation can be found at https://idevision.net/docs\n"
                       f"To regenerate your idevision token, use `{ctx.invoked_with} token`\n"
                       f"To apply for an idevision API token, use `{ctx.invoked_with} apply`")

    @idevision.command("token")
    async def api_token(self, ctx):
        """
        Regenerates your idevision token, and dms it to you.
        """
        async with self.session.post(self.url + "internal/users/token", json={
            "discord_id": ctx.author.id
        }) as resp:
            if resp.status == 400:
                pref = await commands.clean_content().convert(ctx, ctx.prefix)
                return await ctx.send(f"You do not have an idevision account. Use `{pref}idevision apply <reason>` to apply for an account.")
            if resp.status != 200:
                return await ctx.send(f"Internal error: {resp.status}, {await resp.text()}")
            else:
                data = await resp.json()
                await ctx.author.send(f"Your idevision token is `{data['token']}`. Use the `idevision token` command to regenerate your token at any time.")
                await ctx.message.add_reaction("thumbsup")

    @idevision.command("apply")
    async def api_apply(self, ctx: Context, *, reason: str):
        """
        Allows you to apply for a token on the idevision API.
        This will give you access to the OCR endpoint, CDN endpoints, along with higher ratelimits.
        Anyone may use this command.
        """
        async with self.session.post(self.url + "internal/users/apply", json={
            "username": ctx.author.name + ctx.author.discriminator, # intentionally not using str() here
            "userid": ctx.author.id,
            "reason": reason,
            "permissions": ["public.ocr", "cdn"]
        }) as resp:
            if resp.status == 403:
                return await ctx.send(f"Failed to make the request (unauthorized): {await resp.text(), resp.reason}")
            elif resp.status != 201:
                return await ctx.send(f"Internal error: {resp.status}, {resp.reason}")
            else:
                emb = discord.Embed(colour=discord.Colour.dark_gold(), title="Application Recieved", description=f"([jump url]({ctx.message.jump_url}))\nReason:\n{reason}")
                emb = emb.set_author(name=str(ctx.author), icon_url=str(cast(discord.Asset, ctx.author.avatar).url)).set_footer(text=str(ctx.author.id))
                emb.timestamp = datetime.datetime.utcnow()
                await self.hook.send(embed=emb)
                await ctx.send("Application recieved. Please make sure your dms are enabled, or you will not recieve your token when your application is accepted.")

    @idevision.command("accept", aliases=['approve'])
    @commands.is_owner()
    async def accept(self, ctx: Context, user: discord.User):
        async with self.session.post(self.url + "internal/users/accept", json={"userid": user.id}) as resp:
            if resp.status != 201:
                return await ctx.send(f"Internal error: {resp.status}, {resp.reason}")
            else:
                data = await resp.json()
                try:
                    await user.send(f"Your idevision token is `{data['token']}`. Use the `idevision token` command to regenerate your token at any time.")
                except:
                    await ctx.send(f"Accepted {user} (failed to dm the user)")
                else:
                    await ctx.send(f"Accepted {user} (user notified through dm)")

    @idevision.command("deny", aliases=['decline'])
    @commands.is_owner()
    async def user_deny(self, ctx: Context, user: discord.User, allow_reapply=False, *, reason):
        async with self.session.post(self.url + "internal/users/deny", json={"userid": user.id, "retry": allow_reapply, "reason": reason}) as resp:
            if resp.status != 204:
                return await ctx.send(f"Internal error: {resp.status}, {resp.reason}")
            else:
                try:
                    await ctx.author.send(
                        f"Your idevision application has been denied. {'you may reapply' if allow_reapply else ''}\nreason: \n```\n{reason}\n```")
                except:
                    await ctx.send(f"Denied {user} (failed to dm the user)")
                else:
                    await ctx.send(f"Denied {user} (user notified through dm)")

    @idevision.group(invoke_without_command=True, aliases=['user'])
    @commands.is_owner()
    async def users(self, ctx: Context, username: Union[discord.User, str], unsafe:bool=False):
        url = yarl.URL(self.url + "internal/users")
        if isinstance(username, discord.User):
            url = url.with_query(username=username.name + username.discriminator, discord_id=username.id)
        else:
            url = url.with_query(username=username)

        async with self.session.get(url) as resp:
            if 200 <= resp.status < 300:
                data = await resp.json()
                if "auth_key" in data and not unsafe:
                    del data['auth_key']

                await ctx.send("\n".join(f"{x}: {v}" for x, v in data.items()))

            elif resp.reason:
                await ctx.send(f"Failed: ({resp.status})" + resp.reason)

            else:
                await ctx.send("Failed. " + str(resp.status))

    @users.command("add")
    @commands.is_owner()
    async def user_add(self, ctx: Context, user: discord.User, *perms):
        async with self.session.post(self.url + "internal/users",
                                     json={"username": user.name + user.discriminator, "permissions": perms, "discord_id": user.id}) as resp:
            if 200 <= resp.status < 300:
                data = await resp.json()
                await ctx.send(f"User has been added, token: {data['token']}")

            elif resp.reason:
                await ctx.send(f"Failed: ({resp.status})" + resp.reason)

            else:
                await ctx.send("Failed. " + str(resp.status))

    @users.command("addperms", aliases=['addperm', 'ap'])
    @commands.is_owner()
    async def user_add_perm(self, ctx: Context, username: Union[discord.User, str], *perms):
        url = yarl.URL(self.url + "internal/users")
        if isinstance(username, discord.User):
            url = url.with_query(username=username.name + username.discriminator, discord_id=username.id)
        else:
            url = url.with_query(username=username)

        async with self.session.get(url) as resp:
            if 200 <= resp.status < 300:
                data = await resp.json()
                _perms = data['permissions']

            else:
                return await ctx.send(f"Failed: ({resp.status})" + cast(str,resp.reason))

        perms = set(perms)
        perms.update(_perms)
        perms = list(perms)

        async with self.session.patch(self.url + "internal/users",
                                     json={"username": username, "permissions": perms}) as resp:
            if 200 <= resp.status < 300:
                await ctx.send(f"User modified successfully")

            elif resp.reason:
                await ctx.send(f"Failed: {resp.status, resp.reason}")

            else:
                await ctx.send("Failed. " + str(resp.status))

    @users.command("removeperm", aliases=['removeperms', 'rmperm', 'rmperms', 'rp'])
    @commands.is_owner()
    async def user_rm_perm(self, ctx: Context, username: Union[discord.User, str], *perms):
        url = yarl.URL(self.url + "internal/users")
        if isinstance(username, discord.User):
            url = url.with_query(username=username.name + username.discriminator, discord_id=username.id)
        else:
            url = url.with_query(username=username)

        async with self.session.get(url) as resp:
            if 200 <= resp.status < 300:
                data = await resp.json()
                _perms = data['permissions']

            else:
                return await ctx.send(f"Failed: ({resp.status})" + cast(str, resp.reason))

        for p in perms:
            if p in _perms:
                _perms.remove(p)

        async with self.session.patch(self.url + "internal/users",
                                     json={"username": username, "permissions": _perms}) as resp:
            if 200 <= resp.status < 300:
                await ctx.send(f"User modified successfully")

            elif resp.reason:
                await ctx.send(f"Failed: {resp.status, resp.reason}")

            else:
                await ctx.send("Failed. " + str(resp.status))

    @users.command("setname")
    @commands.is_owner()
    async def user_set_name(self, ctx: Context, currentname: Union[discord.User, str], new_name: str):
        if isinstance(currentname, discord.User):
            currentname = currentname.name + currentname.discriminator

        async with self.session.patch(self.url + "internal/users",
                                     json={"username": currentname, "new_username": new_name}) as resp:
            if 200 <= resp.status < 300:
                await ctx.send(f"User modified successfully")

            elif resp.reason:
                await ctx.send(f"Failed: {resp.status, resp.reason}")

            else:
                await ctx.send("Failed. " + str(resp.status))

    @users.command()
    @commands.is_owner()
    async def deauth(self, ctx: Context, username: str):
        async with self.session.post(self.url + "internal/users/deauth", json={"username": username}) as resp:
            if 200 <= resp.status < 300:
                await ctx.send("User has been deauthed")

            elif resp.reason:
                await ctx.send(f"Failed: ({resp.status})" + resp.reason)

            else:
                await ctx.send("Failed. " + str(resp.status))

    @users.command()
    @commands.is_owner()
    async def reauth(self, ctx: Context, username: str):
        async with self.session.post(self.url + "internal/users/auth", json={"username": username}) as resp:
            if 200 <= resp.status < 300:
                await ctx.send("User has been reauthed")

            elif resp.reason:
                await ctx.send(f"Failed: ({resp.status})" + resp.reason)

            else:
                await ctx.send("Failed. " + str(resp.status))

    @idevision.group(invoke_without_command=True)
    @commands.is_owner()
    async def cdn(self, ctx):
        await ctx.send_help(ctx.command)

    @cdn.command("stats")
    @commands.is_owner()
    async def cdn_stats(self, ctx: Context, user: Optional[str] = None):
        if user:
            async with self.session.get(self.url + f"cdn/user?username={user}") as resp:
                if 200 <= resp.status < 300:
                    data = await resp.json()
                    await ctx.send(f"Image count: {data['upload_count']}. "
                                   f"Most recent upload: <{data['last_upload']}>")
                else:
                    await ctx.send(f"fetching from the api failed: {resp.reason}")
        else:
            async with self.session.get(self.url + "cdn") as resp:
                if 200 <= resp.status < 300:
                    data = await resp.json()
                    await ctx.send(f"Image count: {data['upload_count']}. Count today: {data['uploaded_today']} "
                                   f"Most recent: <{data['last_upload']}>")

                else:
                    await ctx.send(f"fetching from the api failed: {resp.reason}")

    @cdn.command("purge", hidden=True)
    @commands.is_owner()
    async def cdn_purge(self, ctx: Context, user: str):
        async with self.session.post(self.url + "cdn/purge", json={"username": user}) as resp:
            if 200 <= resp.status < 300:
                await ctx.send(f"Purged {user}'s images")
            else:
                await ctx.send(f"Failed to purge: {resp.reason}")

    def _parse_args(self, args):
        url = None
        node = None
        filename = None
        for arg in args:
            if not arg.strip():
                continue

            if URL_RE.match(arg):
                if url:
                    raise commands.UserInputError("You may only pass 1 URL to upload")
                url = arg
            else:
                if node:
                    raise commands.UserInputError("You may only pass 1 node and 1 filename")
                if filename:
                    node = arg
                else:
                    filename = arg

        return url, node, filename

    @cdn.command("upload", usage="[url or attachment] [filename] [node]")
    async def cdn_upload(self, ctx: Context, *args):
        """
        Uploads an image to the cdn on your behalf. You must have an idevision auth token with the `cdn` permission group.
        """
        async with ctx.typing():
            url, node, filename = self._parse_args(args)

            if not ctx.message.attachments and not url:
                return await ctx.send("Provide an attachment or link to an image")

            async with self.session.get(self.url + f"internal/users?discordid={ctx.author.id}") as resp:
                if resp.status == 204:
                    return await ctx.send("Public CDN posting is coming soon... for now you must have an idevision login to use this command")
                elif resp.status != 200:
                    return await ctx.send(f"Something went wrong... {resp.reason}")

                user = await resp.json()
                if not user['administrator'] and 'cdn' not in user['permissions']:
                    return await ctx.send("Public CDN posting is coming soon... currently you are not authorized to use the cdn")

                if node and not user['administrator'] and 'cdn.manage' not in user['permissions']:
                    return await ctx.send("You do not have permission to select the node")

                if filename and not user['administrator'] and 'cdn.manage' not in user['permissions']:
                    return await ctx.send("You do not have permission to select the filename")

            if url:
                fn = url.split("/")[-1]
                async with self.bot.session.get(url, headers={"Accept": "image/*"}) as resp:
                    if resp.content_length > 30000000:
                        return await ctx.send("target URL file is too large")

                    fp = io.BytesIO()
                    fp.write(await resp.read())
                    fp.seek(0)

            else:
                fp = io.BytesIO()
                await ctx.message.attachments[0].save(fp)
                fp.seek(0)
                fn = ctx.message.attachments[0].filename

            data = aiohttp.FormData()
            data.add_field('file',
                           fp,
                           filename=fn)

            url = yarl.URL(self.url + "cdn")
            if node and filename:
                url = url.with_query(node=node, name=filename)
            elif node:
                url = url.with_query(node=node)
            elif filename:
                url = url.with_query(name=filename)

            async with self.session.post(url, data=data, headers={"Authorization": user['auth_key'], "File-Name": fn}) as resp:
                if 200 <= resp.status < 300:
                    d = await resp.json()
                    return await ctx.send(f"Uploaded {fn} to the cdn, at {d['url']}")

                else:
                    await ctx.send(f"There was an error uploading the file: ({resp.status}) {resp.reason}")

    @cdn.command("delete", hidden=True)
    async def cdn_delete(self, ctx: Context, url: str):
        """
        deletes an image from the cdn.
        You may only delete your own images unless you have the `cdn.manage` permission group
        """
        try:
            url_ = yarl.URL(url)
            if url_.host != "cdn.idevision.net":
                return await ctx.send("Not a valid image")
            pth = url_.path.strip("/")
            node, image = pth.split("/")
        except:
            return await ctx.send("Not a valid image")

        async with self.session.get(self.url + f"internal/users?discordid={ctx.author.id}") as resp:
            if resp.status == 204:
                return await ctx.send("You are not authorized to use this command")
            elif resp.status != 200:
                return await ctx.send(f"Something went wrong... {resp.reason}")

            user = await resp.json()
            if not user['administrator'] and 'cdn' not in user['permissions']:
                return await ctx.send("You are not authorized to use this command")

        async with self.session.delete(f"{self.url}cdn/{node}/{image}", headers={"Authorization": user['auth_key']}) as resp:
            if resp.status == 404:
                return await ctx.send("Image not found")

            elif resp.status == 401:
                return await ctx.send(resp.reason)

            elif resp.status == 204:
                return await ctx.send("Image deleted successfully")

            else:
                return await ctx.send(f"Something went wrong: {resp.status}, {resp.reason}")

    @cdn.command("listnode")
    @commands.is_owner()
    async def cdn_listnode(self, ctx: Context, node: str):
        """
        Lists the content of a node
        """
        async with self.session.get(self.url + f"cdn/list?node={node}&sort=nodename") as resp:
            if resp.status != 200:
                return await ctx.send(f"Something fucked up: {resp.status} ({resp.reason})")

            data = await resp.json()
            d = [list(x.values()) for x in data[node]]
            d = tabulate.tabulate(d, headers=list(data[node][-1].keys()), tablefmt="psql")
            pages = paginator.TextPages(ctx, d)
            await pages.paginate()

    @cdn.command("nodes")
    @commands.is_owner()
    async def cdn_nodes(self, ctx: Context, safe=True):
        """
        Lists all nodes
        """
        async with self.session.get(self.url + f"cdn/nodes?safe={safe}") as resp:
            if resp.status != 200:
                return await ctx.send(f"Something fucked up: {resp.status} ({resp.reason})")

            data = await resp.json()

        data = sorted({int(x): y for x, y in data.items()}.items(), key=lambda x: x[0])
        fmt = ""

        for (id, node) in data:
            fmt += f"__Node {id}__\n- Name: {node['name']}\n- Port: {node['port']}\n- Last Contact: {round(node['signin'])} Seconds Ago\n"
            if 'ip' in node:
                fmt += f"- IP: {node['ip']}\n"

            fmt += "\n"

        pages = paginator.TextPages(ctx, fmt, prefix="", suffix="")
        await pages.paginate()

    @idevision.command("homepage")
    async def cdn_homepage(self, ctx: Context,
                       display_name,
                       link1="", link1name="",
                       link2="", link2name="",
                       link3="", link3name="",
                       link4="", link4name=""):
        """
        Allows you to set homepage links for https://idevision.net/homepage
        To access your homepage, go to https://idevision.net/homepage?user={ctx.author.name}
        """
        resp = await self.session.post(self.url + "homepage", json={
            "display_name": display_name,
            "user": ctx.author.name,
            "link1": link1,
            "link2": link2,
            "link3": link3,
            "link4": link4,
            "link1_name": link1name,
            "link2_name": link2name,
            "link3_name": link3name,
            "link4_name": link4name,
        })
        resp.close()
        await ctx.send(f"Page is now available at {self.url.replace('api/', 'homepage')}?user={ctx.author.name}")

    @idevision.command("logs")
    @commands.is_owner()
    async def get_logs(self, ctx: Context, unsafe: Optional[bool], page: Optional[int] = 0, oldest_first: Optional[bool]=False):
        url = yarl.URL(self.url+"internal/logs").with_query({
            "page": str(page),
            "oldest-first": str(oldest_first).lower(),
            "safe": str(unsafe if unsafe is not None else True).lower()
        })
        async with self.session.get(url) as resp:
            if resp.status != 200:
                return await ctx.send("Bad Response")

            data = await resp.json()
            data = data['rows']

        table = tabulate.tabulate([list(x.values()) for x in data], headers=list(data[0].keys()))
        pages = paginator.TextPages(ctx, table)
        await pages.paginate()

    @idevision.group("permissions", aliases=['perms'], invoke_without_command=True)
    @commands.is_owner()
    async def perms(self, ctx):
        async with self.session.get(self.url + "internal/permissions") as resp:
            if resp.status != 200:
                return await ctx.send(f"Bad Response {resp.status, resp.reason}")

            await ctx.send(", ".join((await resp.json())['permissions']))

    @perms.command("routes")
    @commands.is_owner()
    async def routes(self, ctx):
        async with self.session.get(self.url + "internal/routes") as resp:
            if resp.status != 200:
                return await ctx.send(f"Bad Response {resp.status, resp.reason}")

            await ctx.paginate_text(pprint.pformat(await resp.json()))

    @perms.command("add")
    @commands.is_owner()
    async def routes_add(self, ctx: Context, perm: str):
        async with self.session.post(self.url + "internal/permissions", json={"permission": perm}) as resp:
            if resp.status != 204:
                return await ctx.send(f"Failed to add permission: {resp.status, resp.reason}")

            await ctx.send("Success")

    @perms.command("remove")
    @commands.is_owner()
    async def routes_remove(self, ctx: Context, perm: str):
        async with self.session.delete(self.url + "internal/permissions", json={"permission": perm}) as resp:
            if resp.status != 204:
                return await ctx.send(f"Failed to add permission: {resp.status, resp.reason}")

            await ctx.send("Success")

    @perms.command(aliases=['removeroute', 'editroute'])
    @commands.is_owner()
    async def addroute(self, ctx: Context, route: str, method: str, perm: Optional[str] = None, force = False):
        if all((route, method, perm)):
            async with self.session.post(self.url + "internal/routes?force=" + str(force), json={
                "endpoint": route,
                "method": method,
                "permission": perm
            }) as resp:
                if resp.status != 204:
                    return await ctx.send(f"Failed to add route: {resp.status, resp.reason}")

                await ctx.send("Success")

        else:
            async with self.session.delete(self.url + "internal/routes", json={
                "endpoint": route,
                "method": method
            }) as resp:
                if resp.status != 204:
                    return await ctx.send(f"Failed to remove the route: {resp.status, resp.reason}")

                await ctx.send("Success")

    @idevision.command("beta")
    @commands.is_owner()
    async def toggle_beta(self, ctx: Context, state: Optional[bool] = None):
        if state is None:
            return await ctx.send(f"Currently targeting {'beta' if self.url == 'https://beta.idevision.net/api/' else 'production'} idevision site ({self.url.replace('api/', '')})")

        if state:
            self.url = "https://beta.idevision.net/api/"
        else:
            self.url = "https://idevision.net/api/"

        await ctx.send(f"Now targeting {'beta' if self.url == 'https://beta.idevision.net/api/' else 'production'} idevision site ({self.url.replace('api/', '')})")

    @commands.command()
    async def retry(self, ctx: Context):
        """
        yeet
        """
        msg: discord.Message = ctx.message
        if not msg.reference:
            return await ctx.send("Missing a reply")

        if msg.reference.cached_message:
            if not await self.bot.is_owner(ctx.author) and msg.reference.cached_message.author.id != ctx.author.id:
                return await ctx.send("Fuck off")

            await self.bot.process_commands(msg.reference.cached_message)

        else:
            data = await ctx.channel.fetch_message(cast(int, msg.reference.message_id))
            if not await self.bot.is_owner(ctx.author) and data.author.id != ctx.author.id:
                return await ctx.send("Fuck off")

            await self.bot.process_commands(data)


import pickle
from collections import Counter
from itertools import chain

import numpy as np

from data.dataimport import import_data
from data.featuredict import FeatureDictionary
from encoders.baseencoder import AbstractEncoder

class TfidfEncoder(AbstractEncoder):
    def decoder_loss(self, data: tuple, representation: np.array) -> float:
        raise Exception("TfidfEncoder has no decoder loss")

    def get_representation_vector_size(self) -> int:
        return len(self.__feature_dict)

    def get_encoding(self, data: tuple) -> np.array:
        document_tokens = Counter(self.__feature_dict.get_id_or_unk(t) for t in data[0])
        vect = np.zeros(len(self.__feature_dict), dtype=np.float)
        for word_id, count in document_tokens.items():
            vect[word_id] = count * self.__idfs[word_id]
        vect /= len(data[0])
        return vect

    def __init__(self, train_file):
        data = import_data(train_file)

        def document_tokens():
            for snippet in data.values():
                yield snippet['original'][0]

        all_document_tokens = [s for s in document_tokens()]
        self.__feature_dict = FeatureDictionary.get_feature_dictionary_for(chain(*all_document_tokens),
                                                                           count_threshold=10)

        self.__idfs = np.ones(len(self.__feature_dict), dtype=np.int)  # use 1s for smoothing
        for document in all_document_tokens:
            document_word_ids = set(self.__feature_dict.get_id_or_unk(t) for t in document)
            for word_id in document_word_ids:
                self.__idfs[word_id] += 1

        self.__idfs = np.log(self.__idfs.astype(np.float))

    def save(self, filename):
        with open(filename, 'wb') as f:
            pickle.dump(self, f, pickle.HIGHEST_PROTOCOL)