import json
import logging
import os
from qgis.PyQt import QtWidgets
from qgis.PyQt import uic

# This loads your .ui file so that PyQt can populate your plugin with the elements from Qt Designer
from PyQt5 import QtCore
from PyQt5.QtCore import QVariant, QRunnable, pyqtSlot, QThreadPool
from PyQt5.QtGui import QIcon, QMovie
from PyQt5.QtWidgets import QComboBox, QHeaderView, QMessageBox, QCheckBox
from qgis._core import QgsMessageLog, Qgis, QgsFields, QgsField, QgsVectorLayer, QgsFeature, QgsGeometry, \
    QgsVectorFileWriter, QgsProject

from .kwg_sparqlquery import kwg_sparqlquery
from .kwg_sparqlutil import kwg_sparqlutil
from .kwg_util import kwg_util as UTIL

FORM_CLASS, _ = uic.loadUiType(os.path.join(
    os.path.dirname(__file__), 'kwg_plugin_enrichment_dialog_base.ui'))

class kwg_pluginEnrichmentDialog(QtWidgets.QDialog, FORM_CLASS):
    def __init__(self, parent=None, s2Cells=[], entityLi=[], spoDict={}, params={}):
        """Constructor."""
        super(kwg_pluginEnrichmentDialog, self).__init__(parent)
        # Set up the user interface from Designer through FORM_CLASS.
        # After self.setupUi() you can access any designer object by doing
        # self.<objectname>, and you can use autoconnect slots - see
        # http://qt-project.org/doc/qt-4.8/designer-using-a-ui-file.html
        # #widgets-and-dialogs-with-auto-connect
        self.setupUi(self)

        # logging
        self.logger = logging.getLogger()
        self.labelPropDict = dict()
        self.labelPropDict["LITERAL"] = "LITERAL"
        self.path = os.path.realpath(
            os.path.join(os.getcwd(), os.path.dirname(__file__)))

        if not os.path.exists(os.path.join(self.path, 'logs')):
            os.makedirs(os.path.join(self.path, 'logs'))
        handler = logging.FileHandler(os.path.join(self.path, 'logs', 'kwg_geoenrichment.log'), 'w+', 'utf-8')
        formatter = logging.Formatter(
            '%(asctime)s - %(levelname)s - %(filename)s:%(funcName)s - %(message)s')
        handler.setFormatter(formatter)  # Pass handler as a parameter, not assign
        self.logger.addHandler(handler)

        self.params = {}
        self.spoDict = spoDict
        self.s2Cells = s2Cells
        self.EntityLi = entityLi
        self.params=params
        if bool(params):
            self.ifaceObj = self.params["ifaceObj"]

        self.threadpool = QThreadPool()

        self.degreeCount = 0
        self.setUpTable()
        self.pushButton_learnMore.clicked.connect(self.addLearnMore)

        # displaying help
        self.displayingHelp = False
        self.setFixedWidth(620)
        self.plainTextEdit.setHidden(True)
        self.plainTextEdit.setReadOnly(True)

        self.toolButton.setIcon(QIcon(":/plugins/kwg_geoenrichment/resources/help-circle.png"))
        self.toolButton.setStyleSheet("background-color: transparent; color: #ffffff; border: none;")
        self.toolButton.setIconSize(QtCore.QSize(32, 32))

        self.movie = QMovie(":/plugins/kwg_geoenrichment/resources/loading.gif")
        self.loadingLabel.setMovie(self.movie)
        self.movie.start()
        self.loadingLabel.setScaledContents(True)
        self.loadingLabel.resize(27, 27)

        self.chosenVal = {}
        self.chosenVal["s"] = {}
        self.chosenVal["p"] = {}
        self.chosenVal["o"] = {}

        self.entitiesRetrieved = QCheckBox()
        self.entitiesRetrieved.setChecked(False)
        self.entitiesRetrieved.stateChanged.connect(lambda: self.populateFirstDegreeSubject())

        self.retrievingQuery = QCheckBox()
        self.retrievingQuery.setChecked(False)
        self.retrievingQuery.stateChanged.connect(lambda: self.queryStateHandler())

        self.toolButton.clicked.connect(self.displayHelp)

        self.sparql_query = kwg_sparqlquery()
        self.sparql_util = kwg_sparqlutil()

        if bool(self.spoDict):
            self.populateFirstDegreeSubject(spo=self.spoDict)

        stylesheet = os.path.join(self.path, 'style.qss')
        fo = open(stylesheet, "r")
        self.setStyleSheet(fo.read())
        fo.close()

    def setParams(self, params):
        self.params.update(params)
        self.ifaceObj = self.params["ifaceObj"]

    def queryStateHandler(self):
        worker = Worker(self.retrievingQuery, self.fetchingLabel, self.loadingLabel)
        self.threadpool.start(worker)

    def setUpTable(self):
        self.tableWidget.setColumnCount(3)
        self.tableWidget.verticalHeader().setVisible(False)
        self.tableWidget.horizontalHeader().setSectionResizeMode(QHeaderView.Stretch)

        self.tableWidget.horizontalHeader().hide()

        self.addLearnMore()

    def addLearnMore(self):

        # populate N degree subject based on the object property above
        if self.degreeCount > 0:
            finalObject = self.tableWidget.cellWidget(self.degreeCount - 1, 2).currentText()

            if finalObject is not None and finalObject != "--- SELECT ---" and finalObject != "LITERAL":
                self.tableWidget.insertRow(self.degreeCount)

                comboBox_S = QComboBox()
                comboBox_P = QComboBox()
                comboBox_O = QComboBox()

                self.tableWidget.setCellWidget(self.degreeCount, 0, comboBox_S)
                self.tableWidget.setCellWidget(self.degreeCount, 1, comboBox_P)
                self.tableWidget.setCellWidget(self.degreeCount, 2, comboBox_O)

                comboBox_S.show()
                comboBox_P.show()
                comboBox_O.show()

                self.tableWidget.cellWidget(self.degreeCount, 0).clear()
                self.tableWidget.cellWidget(self.degreeCount, 0).addItem("--- SELECT ---")
                self.spoDict[self.degreeCount] = {}
                self.spoDict[self.degreeCount]["s"] = {}
                for key in self.spoDict[self.degreeCount - 1]["o"]:
                    self.tableWidget.cellWidget(self.degreeCount, 0).addItem(self.updateLabelPropDict(self.sparql_util.make_prefixed_iri(key)))
                    self.spoDict[self.degreeCount]["s"][key] = self.spoDict[self.degreeCount - 1]["o"][key]
                index = self.tableWidget.cellWidget(self.degreeCount - 1, 2).findText(finalObject, QtCore.Qt.MatchFixedString)

                if index >= 0:
                    self.tableWidget.cellWidget(self.degreeCount, 0).setCurrentIndex(index)
                self.populateNDegreePredicate()
                self.degreeCount += 1
            else:
                msg = QMessageBox()
                msg.setIcon(QMessageBox.Warning)

                msg.setText("Can't add more content...")
                msg.setInformativeText("'LITERAL' data encountered!")
                msg.setWindowTitle("Learn More Warning!")
                msg.exec_()
        else:
            self.tableWidget.insertRow(self.degreeCount)

            comboBox_S = QComboBox()
            comboBox_P = QComboBox()
            comboBox_O = QComboBox()

            self.tableWidget.setCellWidget(self.degreeCount, 0, comboBox_S)
            self.tableWidget.setCellWidget(self.degreeCount, 1, comboBox_P)
            self.tableWidget.setCellWidget(self.degreeCount, 2, comboBox_O)

            comboBox_S.show()
            comboBox_P.show()
            comboBox_O.show()

            self.tableWidget.cellWidget(self.degreeCount, 0).clear()

            self.degreeCount += 1

    def execute(self):
        # manage first degree
        self.results = {}
        self.retrievingQuery.setChecked(True)

        try:
            self.tableWidget.cellWidget(0, 0).currentIndexChanged.disconnect()
        except TypeError:
            pass
        self.tableWidget.cellWidget(0, 0).clear()
        entityBindingObject = []

        for wkt in self.params["wkt_literal"]:
            entityBindingObject.extend(self.sparql_query.getEntityValuesFromGeometry(sparql_endpoint=self.params["end_point"],
                                                                           wkt_literal=wkt))

        for obj in entityBindingObject:
            try:
                self.EntityLi.append(obj["entity"]["value"])
            except Exception as e:
                continue

        self.entitiesRetrieved.setChecked(True)
        self.retrievingQuery.setChecked(False)
        # self.populateFirstDegreeSubject()
        return

    def get_results(self):
        return self.results

    def get_s2Cells(self):
        return self.s2Cells

    def get_entityLi(self):
        return self.EntityLi

    def get_spoDict(self):
        spo = {}
        spo[0] = {}
        spo[0]["s"] = self.spoDict[0]["s"]
        return spo

    def populateFirstDegreeSubject(self, spo={}):
        # self.entitiesRetrieved.stateChanged.disconnect()
        try:
            self.tableWidget.cellWidget(0, 0).currentIndexChanged.disconnect()
        except TypeError:
            pass
        self.retrievingQuery.setChecked(True)
        self.tableWidget.cellWidget(0, 0).clear()
        self.tableWidget.cellWidget(0, 0).addItem("--- SELECT ---")

        if not bool(spo):
            classObject = self.sparql_query.getFirstDegreeClass(sparql_endpoint=self.params["end_point"],
                                                                entityList=self.EntityLi)

            # populate the spoDict
            self.spoDict[0] = {}
            self.spoDict[0]["s"] = {}
            for obj in classObject:
                self.spoDict[0]["s"][obj["type"]["value"]] = self.sparql_util.make_prefixed_iri(obj["label"]["value"])
        else:
            self.spoDict[0] = spo[0]

        for key in self.spoDict[0]["s"]:
            self.tableWidget.cellWidget(0, 0).addItem(self.updateLabelPropDict(self.sparql_util.make_prefixed_iri(key)))
        self.retrievingQuery.setChecked(False)
        self.tableWidget.cellWidget(0, 0).currentIndexChanged.connect(lambda: self.populateFirstDegreePredicate())

    def handleError(self, errCode = ""):
        QgsMessageLog.logMessage("error occured " + errCode, "kwg_geoernichment", Qgis.Info)

    def populateFirstDegreePredicate(self):
        self.tableWidget.cellWidget(0, 1).clear()
        self.tableWidget.cellWidget(0, 1).addItem("--- SELECT ---")
        self.retrievingQuery.setChecked(True)

        self.sub0 = self.labelPropDict[self.tableWidget.cellWidget(0, 0).currentText()]
        self.chosenVal["s"][0] = self.sub0
        predicateObject = self.sparql_query.getFirstDegreePredicate(sparql_endpoint=self.params["end_point"],
                                                                    entityList=self.EntityLi,
                                                                    firstDegreeClass=self.sub0)

        # populate the spoDict
        self.spoDict[0]["p"] = {}
        for obj in predicateObject:
            if "label" in obj:
                self.spoDict[0]["p"][obj["p"]["value"]] = self.sparql_util.make_prefixed_iri(obj["label"]["value"])
            else:
                self.spoDict[0]["p"][obj["p"]["value"]] = ""

        for key in self.spoDict[0]["p"]:
            self.tableWidget.cellWidget(0, 1).addItem(self.updateLabelPropDict(self.sparql_util.make_prefixed_iri(key)))

        self.retrievingQuery.setChecked(False)
        self.tableWidget.cellWidget(0, 1).currentIndexChanged.connect(self.populateFirstDegreeObject)
        return

    def populateFirstDegreeObject(self):
        self.pred0 = self.labelPropDict[self.tableWidget.cellWidget(0, 1).currentText()]
        self.chosenVal["p"][0] = self.pred0
        self.tableWidget.cellWidget(0, 2).clear()
        self.tableWidget.cellWidget(0, 2).addItem("--- SELECT ---")
        self.retrievingQuery.setChecked(True)

        firstDegreeObject = self.sparql_query.getFirstDegreeObject(sparql_endpoint=self.params["end_point"],
                                                                   entityList=self.EntityLi,
                                                                   firstDegreeClass=self.sub0,
                                                                   firstDegreePredicate=self.pred0)

        self.logger.info("firstDegreeObject : ")
        self.logger.info(json.dumps(firstDegreeObject))

        # populate the spoDict
        self.spoDict[0]["o"] = {}
        for obj in firstDegreeObject:
            if "label" in obj:
                self.spoDict[0]["o"][obj["type"]["value"]] = self.sparql_util.make_prefixed_iri(obj["label"]["value"])
            else:
                self.spoDict[0]["o"][obj["type"]["value"]] = ""

        for key in self.spoDict[0]["o"]:
            self.tableWidget.cellWidget(0, 2).addItem(self.updateLabelPropDict(self.sparql_util.make_prefixed_iri(key)))

        self.retrievingQuery.setChecked(False)
        if not self.spoDict[0]["o"]:
            self.tableWidget.cellWidget(0, 2).addItem("LITERAL")

    def populateNDegreePredicate(self):
        i = self.degreeCount
        self.tableWidget.cellWidget(i, 1).clear()
        self.tableWidget.cellWidget(i, 1).addItem("--- SELECT ---")
        selectedVal = []
        for it in range(i+1):
            selectedVal.append(self.labelPropDict[self.tableWidget.cellWidget(it, 0).currentText()])
            if it < i:
                selectedVal.append(self.labelPropDict[self.tableWidget.cellWidget(it, 1).currentText()])
        # secondPropertyURLList.extend(self.getSecondDegreeProperty())
        secondPredObj = self.sparql_query.getNDegreePredicate(sparql_endpoint=self.params["end_point"],
                                                              entityList=self.EntityLi,
                                                              selectedVals=selectedVal,
                                                              degree=i + 1)

        # populate the spoDict
        self.spoDict[i]["p"] = {}
        for obj in secondPredObj:
            if "label" in obj:
                self.spoDict[i]["p"][obj["p"]["value"]] = self.sparql_util.make_prefixed_iri(obj["label"]["value"])
            else:
                self.spoDict[i]["p"][obj["p"]["value"]] = ""

        for key in self.spoDict[i]["p"]:
            self.tableWidget.cellWidget(i, 1).addItem(self.updateLabelPropDict(self.sparql_util.make_prefixed_iri(key)))
        self.tableWidget.cellWidget(i, 1).currentIndexChanged.connect(self.populateNDegreeObject)
        return

    def populateNDegreeObject(self):
        i = self.degreeCount
        self.tableWidget.cellWidget(i - 1, 2).clear()
        self.tableWidget.cellWidget(i - 1, 2).addItem("--- SELECT ---")

        selectedVal = []
        for it in range(i):
            selectedVal.append(self.labelPropDict[self.tableWidget.cellWidget(it, 0).currentText()])
            selectedVal.append(self.labelPropDict[self.tableWidget.cellWidget(it, 1).currentText()])
        # secondPropertyURLList.extend(self.getSecondDegreeProperty())
        secondPropObj = self.sparql_query.getNDegreeObject(sparql_endpoint=self.params["end_point"],
                                                           entityList=self.EntityLi,
                                                           selectedVals=selectedVal,
                                                           degree=i)

        # populate the spoDict
        self.spoDict[i - 1]["o"] = {}
        for obj in secondPropObj:
            if "label" in obj:
                self.spoDict[i - 1]["o"][obj["type"]["value"]] = self.sparql_util.make_prefixed_iri(obj["label"]["value"])
            else:
                self.spoDict[i - 1]["o"][obj["type"]["value"]] = ""

        for key in self.spoDict[i - 1]["o"]:
            self.tableWidget.cellWidget(i - 1, 2).addItem(self.updateLabelPropDict(self.sparql_util.make_prefixed_iri(key)))

        if not self.spoDict[i - 1]["o"]:
            self.tableWidget.cellWidget(i - 1, 2).addItem("LITERAL")

    def getDegree(self):
        return self.degreeCount

    def getResults(self):

        i = self.degreeCount
        selectedVal = []
        for it in range(i):
            subject = self.labelPropDict[self.tableWidget.cellWidget(it, 0).currentText()]
            predicate = self.labelPropDict[self.tableWidget.cellWidget(it, 1).currentText()]
            if subject is not None and subject != "--- SELECT ---" and subject != "LITERAL":
                selectedVal.append(subject)
            if predicate is not None and predicate != "--- SELECT ---" and predicate != "LITERAL":
                selectedVal.append(predicate)

        finalObject = self.labelPropDict[self.tableWidget.cellWidget(i-1, 2).currentText()]
        if finalObject is not None and finalObject != "--- SELECT ---" and finalObject != "LITERAL":
            selectedVal.append(finalObject)

        self.logger.debug("selectedVal + " + str(selectedVal))

        finalResultsObj = self.sparql_query.getNDegreeResults(sparql_endpoint=self.params["end_point"],
                                                           entityList=self.EntityLi,
                                                           selectedVals=selectedVal,
                                                           degree=i)

        self.logger.debug(str(len(finalResultsObj)))
        return finalResultsObj

    def displayHelp(self):
        if self.displayingHelp:
            self.displayingHelp = False
            self.plainTextEdit.setHidden(True)
            self.setFixedWidth(620)
        else:
            self.displayingHelp = True
            self.plainTextEdit.setVisible(True)
            self.setFixedWidth(900)

    def updateLabelPropDict(self, label):
        newLabel = ""
        if ":" in label:
            newLabel = (label.split(":")[1]).strip()
        else:
            newLabel = label
        self.labelPropDict[newLabel] = label
        return newLabel

class Worker(QRunnable, ):

    def __init__(self, retrievingQuery, fetchingLabel, loadingLabel):
        super(Worker, self).__init__()
        self.retrievingQuery = retrievingQuery
        self.loadingLabel = loadingLabel
        self.fetchingLabel = fetchingLabel

    @pyqtSlot()
    def run(self):
        '''
        Runnable code for executing S2 cells query
        '''
        if (self.retrievingQuery.isChecked()):
            # QgsMessageLog.logMessage("fetching...", "kwg_geoenrichment", Qgis.Info)
            self.fetchingLabel.show()
            self.loadingLabel.show()
        else:
            # QgsMessageLog.logMessage("done!", "kwg_geoenrichment", Qgis.Info)
            self.fetchingLabel.hide()
            self.loadingLabel.hide()


# import prodigy
# from prodigy.components.preprocess import add_tokens
# import requests
# import spacy

# @prodigy.recipe("summary-error-analysis")
# def summary_error_analysis(dataset, lang="en"):
#     # We can use the blocks to override certain config and content, and set
#     # "text": None for the choice interface so it doesn't also render the text
#     blocks = [
#         # {"view_id": "ner_manual"},
#         # {"view_id": "choice", "text": None},
#         # {"view_id": "text_input", "field_rows": 3, "field_label": "Explain your decision"}
#         {"view_id": "html"},
#         {"view_id": "spans_manual"},
#     ]

#     def get_stream():
#         res = requests.get("https://cat-fact.herokuapp.com/facts").json()
#         for fact in res["all"]:
#             yield {"text": fact["text"], "options": options}

#     # Load the stream from a JSONL file and return a generator that yields a
#     # dictionary for each example in the data.
#     stream = JSONL(source)

#     # Tokenize the incoming examples and add a "tokens" property to each
#     # example. Also handles pre-defined selected spans. Tokenization allows
#     # faster highlighting, because the selection can "snap" to token boundaries.
#     stream = add_tokens(nlp, stream)

#     nlp = spacy.blank(lang)           # blank spaCy pipeline for tokenization
#     stream = get_stream()             # set up the stream
#     stream = add_tokens(nlp, stream)  # tokenize the stream for ner_manual

#     return {
#         "dataset": dataset,          # the dataset to save annotations to
#         "view_id": "blocks",         # set the view_id to "blocks"
#         "stream": stream,            # the stream of incoming examples
#         "config": {
#             "labels": ["RELEVANT"],  # the labels for the manual NER interface
#             "blocks": blocks         # add the blocks to the config
#         }
#     }

import prodigy
from prodigy.components.loaders import JSONL
from prodigy.components.preprocess import add_tokens
from prodigy.util import split_string
import spacy
from typing import List, Optional

# Recipe decorator with argument annotations: (description, argument type,
# shortcut, type / converter function called on value before it's passed to
# the function). Descriptions are also shown when typing --help.
@prodigy.recipe(
    "summary.manual",
    dataset=("The dataset to use", "positional", None, str),
    spacy_model=("The base model", "positional", None, str),
    source=("The source data as a JSONL file", "positional", None, str),
    label=("One or more comma-separated labels", "option", "l", split_string),
    exclude=("Names of datasets to exclude", "option", "e", split_string),
)
def summary_manual(
    dataset: str,
    spacy_model: str,
    source: str,
    label: Optional[List[str]] = None,
    exclude: Optional[List[str]] = None,
):
    """
    Mark spans manually by token. Requires only a tokenizer and no entity
    recognizer, and doesn't do any active learning.
    """
    # Load the spaCy model for tokenization
    nlp = spacy.load(spacy_model)

    # Load the stream from a JSONL file and return a generator that yields a
    # dictionary for each example in the data.
    stream = JSONL(source)

    # Tokenize the incoming examples and add a "tokens" property to each
    # example. Also handles pre-defined selected spans. Tokenization allows
    # faster highlighting, because the selection can "snap" to token boundaries.
    stream = add_tokens(nlp, stream)

    # We can use the blocks to override certain config and content, and set
    # "text": None for the choice interface so it doesn't also render the text
    blocks = [
        # {"view_id": "ner_manual"},
        # {"view_id": "choice", "text": None},
        # {"view_id": "text_input", "field_rows": 3, "field_label": "Explain your decision"}
        {"view_id": "html"},
        {"view_id": "spans_manual"},
    ]

    return {
        "view_id": "blocks",  # Annotation interface to use
        "dataset": dataset,  # Name of dataset to save annotations
        "stream": stream,  # Incoming stream of examples
        "exclude": exclude,  # List of dataset names to exclude
        "config": {  # Additional config settings, mostly for app UI
            "lang": nlp.lang,
            "labels": label,  # Selectable label options
            "blocks": blocks         # add the blocks to the config
        },
    }


import cv2
import numpy as np
import tensorflow as tf

from tensorflow.keras.layers import Input

try:
    from src.yolo3.model import *
    from src.yolo3.detect import *

    from src.utils.image import *
    from src.utils.datagen import *
    from src.utils.fixes import *
except:
    from script.src.yolo3.model import *
    from script.src.yolo3.detect import *

    from script.src.utils.image import *
    from script.src.utils.datagen import *
    from script.src.utils.fixes import *
import base64

fix_tf_gpu()

def prepare_model(approach):
    '''
    Prepare the YOLO model
    '''
    global input_shape, class_names, anchor_boxes, num_classes, num_anchors, model

    # shape (height, width) of the imput image
    input_shape = (416, 416)

    # class names
    if approach == 1:
        class_names = ['H', 'V', 'W']

    elif approach == 2:
        class_names  = ['W','WH','WV','WHV']

    elif approach == 3:
        class_names  = ['W']

    else:
        raise NotImplementedError('Approach should be 1, 2, or 3')

    # anchor boxes
    if approach == 1:
        anchor_boxes = np.array(
            [
            np.array([[ 76,  59], [ 84, 136], [188, 225]]) /32, # output-1 anchor boxes
            np.array([[ 25,  15], [ 46,  29], [ 27,  56]]) /16, # output-2 anchor boxes
            np.array([[ 5,    3], [ 10,   8], [ 12,  26]]) /8   # output-3 anchor boxes
            ],
            dtype='float64'
        )
    else:
        anchor_boxes = np.array(
            [
            np.array([[ 73, 158], [128, 209], [224, 246]]) /32, # output-1 anchor boxes
            np.array([[ 32,  50], [ 40, 104], [ 76,  73]]) /16, # output-2 anchor boxes
            np.array([[ 6,   11], [ 11,  23], [ 19,  36]]) /8   # output-3 anchor boxes
            ],
            dtype='float64'
        )

    # number of classes and number of anchors
    num_classes = len(class_names)
    num_anchors = anchor_boxes.shape[0] * anchor_boxes.shape[1]

    # input and output
    input_tensor = Input( shape=(input_shape[0], input_shape[1], 3) ) # input
    num_out_filters = ( num_anchors//3 ) * ( 5 + num_classes )        # output

    # build the model
    model = yolo_body(input_tensor, num_out_filters)

    # load weights
    weight_path = f'script/model_data/weights/pictor-ppe-v302-a{approach}-yolo-v3-weights.h5'
    model.load_weights( weight_path )

def get_detection(img):
    # save a copy of the img
    act_img = img.copy()

    # shape of the image
    ih, iw = act_img.shape[:2]

    # preprocess the image
    img = letterbox_image(img, input_shape)
    img = np.expand_dims(img, 0)
    image_data = np.array(img) / 255.

    # raw prediction from yolo model
    prediction = model.predict(image_data)

    # process the raw prediction to get the bounding boxes
    boxes = detection(
        prediction,
        anchor_boxes,
        num_classes,
        image_shape = (ih, iw),
        input_shape = (416,416),
        max_boxes = 10,
        score_threshold=0.3,
        iou_threshold=0.45,
        classes_can_overlap=False)

    # convert tensor to numpy
    boxes = boxes[0].numpy()

    # draw the detection on the actual image
    return draw_detection(act_img, boxes, class_names)

def search_items(img):
    prepare_model(approach=1)
    img = base64.b64decode(img)
    img = np.frombuffer(img, np.uint8)
    img = cv2.imdecode(img, cv2.IMREAD_COLOR)
    img = letterbox_image(img, input_shape)
    img = get_detection(img)
    _, img = cv2.imencode('.jpg', img)
    img = base64.b64encode(img)
    return img.decode('utf-8')


import cv2
import numpy as np

def example_41():
    img = np.random.randint(0, 256, size=[2, 4, 3], dtype=np.uint8)
    rst = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    print("img=\n", img)
    print("rst=\n", rst)
    print("像素点(1,0)直接计算得到的值=",
          img[1, 0, 0] * 0.114 + img[1, 0, 1] * 0.587 + img[1, 0, 2] * 0.299)
    print("像素点(1,0)使用公式 cv2.cvtColor()转换值=", rst[1, 0])
    # 像素点(1,0)直接计算得到的值= 141.765 => 四舍五入
    # 像素点(1,0)使用公式 cv2.cvtColor()转换值= 142

def example_42():
    img = np.random.randint(0, 256, size=[2, 4], dtype=np.uint8)
    rst = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR)
    print("img=\n", img)
    print("rst=\n", rst)

def example_43():
    img = np.random.randint(0, 256, size=[2, 4, 3], dtype=np.uint8)
    rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
    bgr = cv2.cvtColor(rgb, cv2.COLOR_RGB2BGR)
    print("img=\n", img)
    print("rgb=\n", rgb)
    print("bgr=\n", bgr)

def example_44():
    lena = cv2.imread("./images/lena_color.tiff")
    gray = cv2.cvtColor(lena, cv2.COLOR_BGR2GRAY)
    bgr = cv2.cvtColor(gray, cv2.COLOR_GRAY2BGR)
    # ==========打印 shape============
    print("lena.shape=", lena.shape)
    print("gray.shape=", gray.shape)
    print("bgr.shape=", bgr.shape)
    print(bgr)
    # ==========显示效果============
    cv2.imshow("lena", lena)
    cv2.imshow("gray", gray)
    cv2.imshow("bgr", bgr)
    cv2.waitKey()
    cv2.destroyAllWindows()

def example_45():
    lena = cv2.imread("./images/lena_color.tiff")
    rgb = cv2.cvtColor(lena, cv2.COLOR_BGR2RGB)
    cv2.imshow("lena", lena)
    cv2.imshow("rgb", rgb)
    cv2.waitKey()
    cv2.destroyAllWindows()

def example_46():
    # =========测试一下 OpenCV 中蓝色的 HSV 模式值=============
    imgBlue = np.zeros([1, 1, 3], dtype=np.uint8)
    imgBlue[0, 0, 0] = 255
    Blue = imgBlue
    BlueHSV = cv2.cvtColor(Blue, cv2.COLOR_BGR2HSV)
    print("Blue=\n", Blue)
    print("BlueHSV=\n", BlueHSV)
    # =========测试一下 OpenCV 中绿色的 HSV 模式值=============
    imgGreen = np.zeros([1, 1, 3], dtype=np.uint8)
    imgGreen[0, 0, 1] = 255
    Green = imgGreen
    GreenHSV = cv2.cvtColor(Green, cv2.COLOR_BGR2HSV)
    print("Green=\n", Green)
    print("GreenHSV=\n", GreenHSV)
    # =========测试一下 OpenCV 中红色的 HSV 模式值=============
    imgRed = np.zeros([1, 1, 3], dtype=np.uint8)
    imgRed[0, 0, 2] = 255
    Red = imgRed
    RedHSV = cv2.cvtColor(Red, cv2.COLOR_BGR2HSV)
    print("Red=\n", Red)
    print("RedHSV=\n", RedHSV)

def example_47():
    img = np.random.randint(0, 256, size=[5, 5], dtype=np.uint8)
    min = 100
    max = 200
    mask = cv2.inRange(img, min, max)
    print("img=\n", img)
    print("mask=\n", mask)

def example_48():
    img = np.ones([5, 5], dtype=np.uint8) * 9
    mask = np.zeros([5, 5], dtype=np.uint8)
    mask[0:3, 0] = 1
    mask[2:5, 2:4] = 1
    roi = cv2.bitwise_and(img, img, mask=mask)
    print("img=\n", img)
    print("mask=\n", mask)
    print("roi=\n", roi)

def example_49():
    opencv = cv2.imread("./images/OpenCV_Logo_with_text.png")
    hsv = cv2.cvtColor(opencv, cv2.COLOR_BGR2HSV)
    cv2.imshow('opencv', opencv)
    # =============指定蓝色值的范围=============
    minBlue = np.array([110, 50, 50])
    maxBlue = np.array([130, 255, 255])
    # 确定蓝色区域
    mask = cv2.inRange(hsv, minBlue, maxBlue)
    # 通过掩码控制的按位与运算，锁定蓝色区域
    blue = cv2.bitwise_and(opencv, opencv, mask=mask)
    cv2.imshow('blue', blue)
    # =============指定绿色值的范围=============
    minGreen = np.array([50, 50, 50])
    maxGreen = np.array([70, 255, 255])
    # 确定绿色区域
    mask = cv2.inRange(hsv, minGreen, maxGreen)
    # 通过掩码控制的按位与运算，锁定绿色区域
    green = cv2.bitwise_and(opencv, opencv, mask=mask)
    cv2.imshow('green', green)
    # =============指定红色值的范围=============
    minRed = np.array([0, 50, 50])
    maxRed = np.array([30, 255, 255])
    # 确定红色区域
    mask = cv2.inRange(hsv, minRed, maxRed)
    # 通过掩码控制的按位与运算，锁定红色区域
    red = cv2.bitwise_and(opencv, opencv, mask=mask)
    cv2.imshow('red', red)
    cv2.waitKey()
    cv2.destroyAllWindows()

def example_410():
    # 标记肤色
    pass

def example_411():
    img = cv2.imread("./images/lena_color.tiff")
    hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
    h, s, v = cv2.split(hsv)
    print(h.shape, s.shape, v.shape)
    v[:, :] = 255
    newHSV = cv2.merge([h, s, v])
    art = cv2.cvtColor(newHSV, cv2.COLOR_HSV2BGR)
    cv2.imshow("img", img)
    cv2.imshow("art", art)
    cv2.waitKey()
    cv2.destroyAllWindows()

def example_412():
    img = np.random.randint(0, 256, size=[2, 3, 3], dtype=np.uint8)
    bgra = cv2.cvtColor(img, cv2.COLOR_BGR2BGRA)
    print("img=\n", img)
    print("bgra=\n", bgra)
    b, g, r, a = cv2.split(bgra)
    print("a=\n", a)
    a[:, :] = 125
    bgra = cv2.merge([b, g, r, a])
    print("bgra=\n", bgra)

def example_413():
    img = cv2.imread("./images/lena_color.tiff")
    bgra = cv2.cvtColor(img, cv2.COLOR_BGR2BGRA)
    b, g, r, a = cv2.split(bgra)
    a[:, :] = 125
    bgra125 = cv2.merge([b, g, r, a])
    a[:, :] = 0
    bgra0 = cv2.merge([b, g, r, a])
    cv2.imshow("img", img)
    cv2.imshow("bgra", bgra)
    cv2.imshow("bgra125", bgra125)
    cv2.imshow("bgra0", bgra0)
    cv2.waitKey()
    cv2.destroyAllWindows()
    # 各个图像的 alpha 通道值虽然不同，但是在显示时是没有差别的

    cv2.imwrite("bgra.png", bgra)
    cv2.imwrite("bgra125.png", bgra125)
    cv2.imwrite("bgra0.png", bgra0)

if __name__ == '__main__':
    # example_41()
    # example_42()
    # example_43()
    example_44()
    # example_45()
    # example_46()
    # example_47()
    # example_48()
    # example_49()
    # example_411()
    # example_412()
    # example_413()


import os
import discord
from discord.ext import commands
from dotenv import load_dotenv
import cartons

load_dotenv()

TOKEN = os.getenv('DISCORD_TOKEN')
GUILD = os.getenv('DISCORD_GUILD')

bot = commands.Bot(command_prefix='!')

@bot.command(name="combien", help="Donne les totaux des cartons")
async def combien(ctx):
    message = []
    for carton in cartons.fethAllCartons():
        user = discord.utils.find(lambda u: u.id == carton.user, bot.users)
        emoji = discord.utils.find(lambda e: e.id == carton.carton, bot.emojis)
        message.append(f'{user.name} à {carton.nb} <:{emoji.name}:{carton.carton}>')
    if len(message) == 0:
        message.append('Aucun carton')
    await ctx.send("\n".join(message))

@bot.command(name="carton", help="Donne les totaux de tes cartons")
async def carton(ctx, pseudo=''):
    if pseudo == '':
        pseudo = ctx.author.name
    u = discord.utils.find(lambda u: u.name == pseudo, bot.users)
    print(f'showing carton pour {pseudo}({u.id})')
    message = []
    for carton in cartons.fethAllCartons():
        if carton.user != u.id:
            continue
        emoji = discord.utils.find(lambda e: e.id == carton.carton, bot.emojis)
        message.append(f'{u.name} à {carton.nb} <:{emoji.name}:{carton.carton}>')
    if len(message) == 0:
        message.append('Aucun carton')
    await ctx.send("\n".join(message))

@bot.event
async def on_ready():
    guild = discord.utils.get(bot.guilds, name=GUILD)
    print(
        f'{bot.user} is connected to the following guild:\n'
        f'{guild.name}(id: {guild.id})'
    )

bot.run(TOKEN)


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import numpy as np
import gin.tf
from disentanglement_lib.evaluation.metrics.beta_vae import compute_beta_vae_sklearn
from disentanglement_lib.evaluation.metrics.dci import _compute_dci, compute_dci
from disentanglement_lib.evaluation.metrics.factor_vae import compute_factor_vae
from disentanglement_lib.evaluation.metrics.mig import _compute_mig, compute_mig
from disentanglement_lib.evaluation.metrics.modularity_explicitness import modularity, explicitness_per_factor, \
    compute_modularity_explicitness
from disentanglement_lib.evaluation.metrics.sap_score import _compute_sap, compute_sap
import pandas as pd
from disentanglement_lib.evaluation.metrics.utils import _histogram_discretize

class GT:
    def __init__(self, discr=False, limit=True):
        d = pd.read_csv("../data/aling.csv")
        self.mu = np.array(d[['X0', 'X1']]).T
        self.y = np.array(d[['beta_untrans', 'persv']]).T

        dmu = self.mu.copy()
        dy = self.y.copy()

        if discr:
            dmu[0] = pd.qcut(dmu[0], q=15, labels=False)
            dmu[1] = pd.qcut(dmu[1], q=15, labels=False)
            self.mu = dmu

        dy[0] = pd.qcut(dy[0], q=15, labels=False)
        dy[1] = pd.qcut(dy[1], q=15, labels=False)
        self.y = dy

        self.num_factors = 2

        self.d = {}
        for i in range(self.y.shape[1]):
            self.d[str(self.y[:, i])] = i

        np.random.seed(00)
        self.cur_sample = np.arange(0, self.y.shape[1])
        np.random.shuffle(self.cur_sample)
        self.cur_index = 0
        self.limit = limit

    def sample(self, n_points, state):
        out = np.zeros((2, n_points))
        i = 0
        cur_i = []
        while i < n_points:
            out[:, i] = self.y[:, self.cur_sample[self.cur_index]]
            cur_i.append(self.cur_sample[self.cur_index])
            self.cur_index += 1
            if not self.limit:
                if self.cur_index >= 1500:
                    self.cur_index = 0
            i += 1

        return out.T, cur_i

    def sample_observations(self, p, r):
        _, o = self.sample(p, r)
        return np.array(o)

    def sample_factors(self, num, random_state):
        return self.sample(num, random_state)[0]

    def sample_observations_from_factors(self, factors, r):
        out = []
        for f in range(factors.shape[0]):
            out.append(self.d[str(factors[f, :])])
        return out

def reps_fun(gt):
    def reprs(obs):
        reps = np.zeros((2, len(obs)))
        for f in range(len(obs)):
            reps[:, f] = gt.mu[:, obs[f]]
        return reps.T
    return reprs

if __name__ == "__main__":
    gin.bind_parameter("discretizer.discretizer_fn", _histogram_discretize)
    gin.bind_parameter("discretizer.num_bins", 10)

    gt = GT(True)
    print(compute_mig(gt, reps_fun(gt), num_train=1500, random_state=2020))
    #
    gt = GT(True)
    print(compute_dci(gt, reps_fun(gt), num_train=1000, random_state=2020, num_test=500))
    #
    gt = GT(True)
    print(compute_sap(gt, reps_fun(gt), num_train=1000, random_state=np.random.RandomState(0), num_test=500, continuous_factors=False))
    #
    gt = GT(True)
    print(compute_modularity_explicitness(gt, reps_fun(gt), num_train=1000, random_state=np.random.RandomState(0), num_test=500))

    gt = GT(False, False)
    print(compute_beta_vae_sklearn(gt, reps_fun(gt), num_train=1000, batch_size=64, num_eval=500,
                                   random_state = np.random.RandomState(0)))
    gt = GT(False, False)
    print(compute_factor_vae(gt, reps_fun(gt), num_train=1000, batch_size=64, num_eval=500,
                                   random_state = np.random.RandomState(0), num_variance_estimate=10))


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models

class Migration(migrations.Migration):

    dependencies = [
        ('gatekeeper', '0038_accountcheckattempt'),
    ]

    operations = [
        migrations.AlterField(
            model_name='user',
            name='last_login',
            field=models.DateTimeField(verbose_name='last login', null=True, blank=True),
        ),
    ]


import csv
import argparse
import re

from hdt import HDTDocument

OWL_CLASS = "http://www.w3.org/2002/07/owl#Class"
RDF_CLASS = "http://www.w3.org/2000/01/rdf-schema#Class"
RDF_TYPE = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type"
LABEL = "http://www.w3.org/2000/01/rdf-schema#label"
COMMENT = "http://www.w3.org/2000/01/rdf-schema#comment"
OCCURENCIES = "http://purl.org/vocommons/voaf#occurrencesInDatasets"
REUSED = "http://purl.org/vocommons/voaf#reusedByDatasets"
SUB_CLASSES = "http://www.w3.org/2000/01/rdf-schema#subClassOf"
EQUIVALENT_CLASSES = "http://www.w3.org/2002/07/owl#equivalentClass"
DEFINED_BY = "http://www.w3.org/2000/01/rdf-schema#isDefinedBy"

OWL_PROPERTY = "http://www.w3.org/2002/07/owl#ObjectProperty"
RDF_PROPERTY = "http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"
DOMAIN = "http://www.w3.org/2000/01/rdf-schema#domain"
RANGE = "http://www.w3.org/2000/01/rdf-schema#range"
SUB_PROPERTY = "http://www.w3.org/2000/01/rdf-schema#subPropertyOf"
EQUIVALENT_PROPERTY = "http://www.w3.org/2000/01/rdf-schema#subPropertyOf"

try:
    LOV = HDTDocument("lov_hdt/lov.hdt")
except RuntimeError:
    raise RuntimeError("LOV dump not found: Put lov dump into lov_hdt/ folder")

def generate_classes_dataset():
    with open('lov_csv/lov_classes.csv', 'w') as f:
        writer = csv.writer(f, delimiter=";")
        writer.writerow(['uri', 'uri_prefix', 'uri_suffix', 'label', 'description', 'language', 'occurencies_in_datasets', 'reused_by_datasets', 'sub_classes', 'equivalent_classes', 'equivalent_classes_suffix', 'defined_by'])
        (classe_triples, cardinality) = LOV.search_triples("", RDF_TYPE, OWL_CLASS)
        (classe_triples2, cardinality2) = LOV.search_triples("", RDF_TYPE, RDF_CLASS)
        hdt_iterators = [classe_triples, classe_triples2]
        already_added_uri = []
        for hdt_iterator in hdt_iterators:
            for classe_triple in hdt_iterator:
                uri_classe = classe_triple[0].encode('utf-8')
                if 'http' in uri_classe and uri_classe not in already_added_uri:
                    uri_prefix = get_uri_prefix(uri_classe)
                    uri_suffix = clean(get_uri_suffix(uri_classe))
                    already_added_uri.append(uri_classe)
                    sub_classes = []
                    equivalent_classes = []
                    equivalent_classes_suffix = [uri_suffix]
                    (sub_triples, cardinality) = LOV.search_triples(uri_classe, SUB_CLASSES, "")
                    for sub_triple in sub_triples:
                        if 'http' in sub_triple[2]:
                            sub_classes.append(sub_triple[2].encode('utf-8'))
                    if sub_classes:
                        sub_classes = ", ".join(sub_classes)
                    else:
                        sub_classes = ''
                    (equi_triples, cardinality) = LOV.search_triples(uri_classe, EQUIVALENT_CLASSES, "")
                    for equi_triple in equi_triples:
                        if 'http' in equi_triple[2]:
                            equivalent_classes.append(equi_triple[2].encode('utf-8'))
                            equivalent_classes_suffix.append(get_uri_suffix(equi_triple[2].encode('utf-8')))
                    if equivalent_classes:
                        equivalent_classes = ", ".join(equivalent_classes)
                    else:
                        equivalent_classes = ''
                    if equivalent_classes_suffix:
                        equivalent_classes_suffix = ", ".join(equivalent_classes_suffix)
                    else:
                        equivalent_classes_suffix = ''
                    (defined_triples, cardinality) = LOV.search_triples(uri_classe, DEFINED_BY, "")
                    defined_by = ''
                    for defined_triple in defined_triples:
                        defined_by = defined_triple[2].replace('"', '').encode('utf-8')
                    (occurences_triples, cardinality) = LOV.search_triples(uri_classe, OCCURENCIES, "")
                    if cardinality > 0:
                        occurencies = int(occurences_triples.next()[2].split('^^')[0].replace('"', ''))
                    else:
                        occurencies = 0
                    (reused_triples, cardinality) = LOV.search_triples(uri_classe, REUSED, "")
                    if cardinality > 0:
                        reused = int(reused_triples.next()[2].split('^^')[0].replace('"', ''))
                    else:
                        reused = 0
                    labels_and_descriptions = {}
                    (label_triples, cardinality) = LOV.search_triples(uri_classe, LABEL, "")
                    for label_triple in label_triples:
                        label, language = split_string_lang(label_triple[2])
                        if language not in labels_and_descriptions:
                            labels_and_descriptions[language] = {}
                        labels_and_descriptions[language]['label'] = label
                    (description_triples, cardinality) = LOV.search_triples(uri_classe, COMMENT, "")
                    for description_triple in description_triples:
                        description, language = split_string_lang(description_triple[2])
                        if language not in labels_and_descriptions:
                            labels_and_descriptions[language] = {}
                        labels_and_descriptions[language]['description'] = description
                    if labels_and_descriptions:
                        for language, value in labels_and_descriptions.iteritems():
                            if 'label' not in value:
                                label = uri_suffix
                            else:
                                label = value['label']
                            if 'description' not in value:
                                description = label
                            else:
                                description = value['description']
                            writer.writerow([uri_classe, uri_prefix, uri_suffix, label, description, language, occurencies, reused, sub_classes, equivalent_classes, equivalent_classes_suffix, defined_by])
                    else:
                        writer.writerow([uri_classe, uri_prefix, uri_suffix, uri_suffix, uri_suffix, 'undefined', occurencies, reused, sub_classes, equivalent_classes, equivalent_classes_suffix, defined_by])

def generate_properties_dataset():
    with open('lov_csv/lov_properties.csv', 'w') as f:
        writer = csv.writer(f, delimiter=";")
        writer.writerow(['uri', 'uri_prefix', 'uri_suffix', 'label', 'description', 'language', 'occurencies_in_datasets', 'reused_by_datasets', 'domain', 'range', 'sub_properties', 'equivalent_properties', 'equivalent_properties_suffix', 'defined_by'])
        (property_triples, cardinality) = LOV.search_triples("", RDF_TYPE, OWL_PROPERTY)
        (property_triples2, cardinality2) = LOV.search_triples("", RDF_TYPE, RDF_PROPERTY)
        hdt_iterators = [property_triples, property_triples2]
        already_added_uri = []
        for hdt_iterator in hdt_iterators:
            for property_triple in hdt_iterator:
                uri_property = property_triple[0].encode('utf-8')
                if 'http' in uri_property and uri_property not in already_added_uri:
                    uri_prefix = get_uri_prefix(uri_property)
                    uri_suffix = clean(get_uri_suffix(uri_property))
                    already_added_uri.append(uri_property)
                    sub_properties = []
                    equivalent_properties = []
                    equivalent_properties_suffix = [uri_suffix]
                    (sub_triples, cardinality) = LOV.search_triples(uri_property, SUB_PROPERTY, "")
                    for sub_triple in sub_triples:
                        if 'http' in sub_triple[2]:
                            sub_properties.append(sub_triple[2].encode('utf-8'))
                    if sub_properties:
                        sub_properties = ", ".join(sub_properties)
                    else:
                        sub_properties = ''
                    (equi_triples, cardinality) = LOV.search_triples(uri_property, EQUIVALENT_PROPERTY, "")
                    for equi_triple in equi_triples:
                        if 'http' in equi_triple[2]:
                            equivalent_properties.append(equi_triple[2].encode('utf-8'))
                            equivalent_properties_suffix.append(get_uri_suffix(equi_triple[2].encode('utf-8')))
                    if equivalent_properties:
                        equivalent_properties = ", ".join(equivalent_properties)
                    else:
                        equivalent_properties = ''
                    if equivalent_properties_suffix:
                        equivalent_properties_suffix = ", ".join(equivalent_properties_suffix)
                    else:
                        equivalent_properties_suffix = ''
                    (defined_triples, cardinality) = LOV.search_triples(uri_property, DEFINED_BY, "")
                    defined_by = ''
                    for defined_triple in defined_triples:
                        defined_by = defined_triple[2].replace('"', '').encode('utf-8')
                    (domain_triples, cardinality) = LOV.search_triples(uri_property, DOMAIN, "")
                    domain = ''
                    for domain_triple in domain_triples:
                        domain = domain_triple[2].replace('"', '').encode('utf-8')
                    (range_triples, cardinality) = LOV.search_triples(uri_property, RANGE, "")
                    p_range = ''
                    for range_triple in range_triples:
                        p_range = range_triple[2].replace('"', '').encode('utf-8')
                    (occurences_triples, cardinality) = LOV.search_triples(uri_property, OCCURENCIES, "")
                    if cardinality > 0:
                        occurencies = int(occurences_triples.next()[2].split('^^')[0].replace('"', ''))
                    else:
                        occurencies = 0
                    (reused_triples, cardinality) = LOV.search_triples(uri_property, REUSED, "")
                    if cardinality > 0:
                        reused = int(reused_triples.next()[2].split('^^')[0].replace('"', ''))
                    else:
                        reused = 0
                    labels_and_descriptions = {}
                    (label_triples, cardinality) = LOV.search_triples(uri_property, LABEL, "")
                    for label_triple in label_triples:
                        label, language = split_string_lang(label_triple[2])
                        if language not in labels_and_descriptions:
                            labels_and_descriptions[language] = {}
                        labels_and_descriptions[language]['label'] = label
                    (description_triples, cardinality) = LOV.search_triples(uri_property, COMMENT, "")
                    for description_triple in description_triples:
                        description, language = split_string_lang(description_triple[2])
                        if language not in labels_and_descriptions:
                            labels_and_descriptions[language] = {}
                        labels_and_descriptions[language]['description'] = description
                    if labels_and_descriptions:
                        for language, value in labels_and_descriptions.iteritems():
                            if 'label' not in value:
                                label = uri_suffix
                            else:
                                label = value['label']
                            if 'description' not in value:
                                description = label
                            else:
                                description = value['description']
                            writer.writerow([uri_property, uri_prefix, uri_suffix, label, description, language, occurencies, reused, domain, p_range, sub_properties, equivalent_properties, equivalent_properties_suffix, defined_by])
                    else:
                        writer.writerow([uri_property, uri_prefix, uri_suffix, uri_suffix, uri_suffix, 'undefined', occurencies, reused, domain, p_range, sub_properties, equivalent_properties, equivalent_properties_suffix, defined_by])

def split_string_lang(obj):
    splitted_obj = obj.replace('"', '').split('@')
    string = splitted_obj[0].replace('^^http://www.w3.org/2001/XMLSchema#string', '').encode('utf-8')
    language = 'undefined'
    if len(splitted_obj) > 1:
        language = splitted_obj[1].encode('utf-8')
    return string, language

def get_uri_suffix(uri):
    if '#' in uri:
        return uri.rsplit('#', 1)[-1]
    else:
        return uri.rsplit('/', 1)[-1]

def get_uri_prefix(uri):
    if '#' in uri:
        return '{}#'.format(uri.rsplit('#', 1)[0])
    else:
        return '{}/'.format(uri.rsplit('/', 1)[0])

first_cap_re = re.compile('(.)([A-Z][a-z]+)')
all_cap_re = re.compile('([a-z0-9])([A-Z])')

def clean(str):
    # UnCamel, UnSnake ...
    s1 = first_cap_re.sub(r'\1_\2', str)
    s2 = all_cap_re.sub(r'\1_\2', s1).lower()
    return s2.replace('_', ' ').replace('-', ' ')

def main():
    parser = argparse.ArgumentParser(prog='lovcsv', description='LOV rdf to csv')
    parser.add_argument('properties_or_classes', metavar='PC', type=str, nargs='+',
                        help='select the csv dataset to generate')
    args = parser.parse_args()
    dataset = args.properties_or_classes[0]
    if dataset == "classes":
        generate_classes_dataset()
    else:
        generate_properties_dataset()

if __name__ == "__main__":
    main()


# a module to contain all spotify spotipy API extract and transform functions

import spotipy
from matplotlib import pyplot as plt
from spotipy.oauth2 import SpotifyClientCredentials

from settings import Settings

def generalAuthSpotify(ap_settings):
	"""Initializes general spotify credentials"""
	scc = SpotifyClientCredentials(ap_settings.client_id, ap_settings.client_secret)
	sp = spotipy.Spotify(client_credentials_manager = scc)

	return sp

def getPitchInfo(urn, ap_settings, sp):
	"""Extracts a tracks pitch and rythm info"""
	#Get Track Info
	track = sp.track(urn)
	t_name = track['name']
	t_artist = track['artists'][0]['name']

	#extract audio analysis segments
	audio = sp.audio_analysis(urn)
	bars = audio['bars']
	beats = audio['beats']
	tatums = audio['tatums']
	sections = audio['sections']
	segments = audio['segments']

	#extract segment pitch details
	#create list of pitches
	pitches = []
	#create dictonary of pitches and x values
	pitch_times = {}
	#create list for pitch x and y values
	pitch_times_x = []
	pitch_times_y = []

	#add pitch segment pitch records to that list
	for segment in segments:
		#read in relevant pitch information
		start = segment['start']
		pitch = segment['pitches']
		num_pitches = len(pitch)
		duration = segment['duration']
		#determine segments pitch divisor
		pitch_divisor = duration/num_pitches
		#define x passer
		x_passer = start

		#determine the dominant pitch based on the 12 pitch classes
		#set pitch class determinant parameters
		index = 0
		max = 0
		pitch_int = 0
		num_ones = 0

		for p in pitch:
			#log if there are more than 1 perfect matches
			if p == 1:
				num_ones =+ 1
			#if there is more than 1 perfect match, terminate the loop and log it
			if num_ones > 1:
				too_noisy = True
				p_int = 99
				break
			else:
				too_noisy = False
			#store the highest probabilty of a pitch class
			if p > max:
				max = p
				pitch_int = index
			else:
				pitch_int = pitch_int

			#keep tracking the index
			index += 1
		y_val = pitch_int
		pitch_times_y.append(y_val)

	#get other rhythm information
	#create x coordinate lists. ts = timestamp
	bar_ts = []
	beat_ts = []
	tatum_ts = []
	section_ts = []
	segment_ts = []
	#create y coordinate lists
	bar_y = []
	beat_y = []
	tatum_y = []
	section_y = []
	segment_y = []
	pitch_y = [] #y coordinate of segments as a function of pitch class integer
	#define y coordinates
	y_bar = ap_settings.y_bar
	y_beat = ap_settings.y_beat
	y_tatum = ap_settings.y_tatum
	y_section = ap_settings.y_section
	y_segment = ap_settings.y_segment

	#get segment coordinates for base layers
	for bar in bars:
		x = bar['start']
		bar_ts.append(x)
		bar_y.append(y_bar)
	for beat in beats:
		x = beat['start']
		beat_ts.append(x)
		beat_y.append(y_beat)
	for tatum in tatums:
		x = tatum['start']
		tatum_ts.append(x)
		tatum_y.append(y_tatum)
	for section in sections:
		x = section['start']
		section_ts.append(x)
		section_y.append(y_section)
	for segment in segments:
		#convert it to minutes
		x = segment['start']
		segment_ts.append(x)
		segment_y.append(y_segment)

	#create a dictionary for the return
	pitchinfo = {'track name': t_name, 'track artist': t_artist, 'bar ts': bar_ts,
				'bar y': bar_y, 'beat ts': beat_ts, 'tatum ts': tatum_ts,
				'tatum y': tatum_y, 'section ts': section_ts, 'section y':
				section_y, 'segment ts': segment_ts, 'segment y': segment_y,
				'pitch values': pitch_times_y}
	#print(pitch_times_y)
	#pitch_ys = pitchinfo['pitch values']
	#print(pitch_ys)
	#return all dectionary of extracted values
	return pitchinfo

def pitchClassToFreq(pitch_info, ap_settings):
	"""
	Function that looks up frequencies for pitch class and converts to
	conventional music notes and frequency values
	"""
	#create pitch list
	pitch_notes = []
	#create frequency list
	pitch_freqs = []

	pitch_list = pitch_info['pitch values']

	for pitch in pitch_list:
		#get pitch class note name
		for name, int in ap_settings.pitch_class.items():
			if int == pitch:
				note = name
				pitch_notes.append(note)
				break
		for name, freq in ap_settings.pitch_freq.items():
			if name == note:
				hz = freq
				pitch_freqs.append(hz)
				break
	#create dictionary for return
	pitchtransformed = {'notes': pitch_notes, 'frequencies': pitch_freqs}

	return pitchtransformed

def getTracks(username, sp):
	""" A function to get tracks and info from a group of playlists"""

	#set limits for testing
	testlimit = 1
	#read in the playlists
	plists = sp.user_playlists(username, testlimit)

	#create a track name dictionary of playlist results
	plistTracks = {}

	#fill dictionary with playlist results
	for plist in plists['items']:
		# Get playlist name
		plist_name = plist['name']
		print(plist_name)

		# Read in playlist specific info
		plist = sp.user_playlist(username, plist['id'],
			fields = 'artist, tracks, name, duration_ms, next')

		# Get the tracks from the playlist
		tracks = plist['tracks']

		# Iterate through the tracks and load track info
		for i, item in enumerate(tracks['items']):
			# Get track info
			track = item['track']
			t_name = track['name']
			t_artist = track['artists'][0]['name']
			t_album = track['album']['name']
			t_dur = track['duration_ms']
			t_pop = track['popularity']
			#get tracks audio features if needed
			if return_dets == 'af_y':
				t_uri = track['uri']
				audio = sp.audio_features(t_uri)[0]
			else:
				audio = [0]

			#convert from ms to s and round to the nearest second
			t_dur_sec = t_dur//1000

			#set dictionary key
			entryname = plist_name + '_' + t_name

			#load info into track dictionary
			track = {'playlist': plist_name, 'name': t_name, 'artist': t_artist,
				'album': t_album, 'duration': t_dur_sec, 'popularity': t_pop,
				'audio_features': audio}
			#update main dictionary
			plistTracks[entryname] = track

		# rest to not trip API limit
		time.sleep(0.5)

	return plistTracks

def findDuplicates(tracks, filepath, filename):
	"""Finds duplicates in a dictinary of tracks"""
	# create a duplicate tracking dictionary
	trackDups = {}

	# iterate through the tracks to find duplicates
	for plist_name, track in tracks.items():
		try:
			name = track['name']
			duration = track['duration']
			artist = track['artist']
			popularity = track['popularity']
			# look for existing entries
			if name in trackDups:
				# if a name and duration match, increment the count
				if duration == trackDups[name][0]:
					#print(name + " " + str(trackDups[duration][0]))
					count = trackDups[name][1]
					trackDups[name] = (duration, count+1, artist, popularity)
			else:
				# add dicitonary entry as tuple (duration, count)
				trackDups[name] = (duration, 1, artist, popularity)
		except:
			# ignore
			pass

	total_duplicates = len(tracks) - len(trackDups)
	print("Total Tracks: " + str(len(trackDups)))
	print("Total Duplicates: " + str(total_duplicates))

	# store duplicates as (name, count) tuples
	dups = []
	for k, v in trackDups.items():
		if v[1] > 1:
			dups.append((v[1], k))
	# save duplicates to a file
	#print("# of duplicate tracks to be saved: " + str(len(dups)))
	if len(dups) > 0:
		print("Found %d duplicates. Track names saved to dup.txt" % len(dups))
	else:
		print("No duplicate tracks found!")
	f = open(filepath + filename, "w")
	for val in dups:
		f.write("[%d] %s\n" % (val[0], val[1]))
	f.close()

def displayAudioFeatures(urn):
	track = sp.track(urn)
	t_name = track['name']
	t_artist = track['artists'][0]['name']
	audio = sp.audio_features(urn)[0]
	#acousticness = audio['acousticness']

	print("Artist: " + t_artist)
	print("Song: " + t_name)
	print("\nAudio Analysis: \n")

	for key, value in audio.items():
		try:
			print(key + ' ' + value)
		except:
			print(key + ' ' + str(value))


import os
from tqdm import tqdm
import cv2
from PIL import Image
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from transformers import TransfoXLCorpus
import helper
import torch
import torch.optim  as optim
import torch.nn as nn
import torchvision
from torchvision import datasets
from torch.utils.data import Dataset,DataLoader
from  torchsummary import summary
import albumentations as A
import segmentation_models_pytorch as smp
from segmentation_models_pytorch.losses import DiceLoss,FocalLoss,JaccardLoss
from sklearn.model_selection import train_test_split

os.chdir('C:/ritika/kaggle_hackathons')

CSV_FILE='road_dataset/train.csv'
DATA_DIR='road_dataset/'

DEVICE='cpu'
EPOCHS=2
LR=0.003
BATCH_SIZE=8
IMG_SIZE=512

ENCODER= 'timm-efficientnet-b0'
WEIGHTS='imagenet'

sample_img= cv2.imread('road_dataset/images/10528675_15.png')
print(sample_img.shape)
df=pd.read_csv(CSV_FILE)

print(df.head())

#idx=20
#row=df.iloc[idx]

#image_path= DATA_DIR + row.images
#mask_path= DATA_DIR + row.masks
# reading the sample image
#image=cv2.imread(image_path)
#image= cv2.cvtColor(image,cv2.COLOR_BGR2RGB)

##mask= cv2.imread(mask_path)
#mask= cv2.cvtColor(mask,cv2.IMREAD_GRAYSCALE)/255

#f,(ax1,ax2)= plt.subplots(1,2, figsize=(10,5))

#ax1.set_title('IMAGE')
#ax1.imshow(image)

#ax2.set_title('GROUND TRUTH')
#ax2.imshow(mask,cmap='gray')

#plt.show()

train_df,valid_df= train_test_split(df,test_size=0.20,random_state=42)
print(len(train_df))
print(len(valid_df))

# Augmentation transforms
def train_augs():
    return A.Compose([A.Resize(IMG_SIZE,IMG_SIZE),A.HorizontalFlip(p=0.5),A.VerticalFlip(p=0.5)])

def valid_augs():
    return A.Compose([A.Resize(IMG_SIZE,IMG_SIZE)])

# create custom dataset
class SegmentationDataset(Dataset):

    def __init__(self,df,augmentations):
        self.df=df
        self.augmentations=augmentations

    def __len__(self):
        return len(self.df)

    def __getitem__(self, idx):
        row=self.df.iloc[idx]
        image_path= DATA_DIR + row.images
        mask_path= DATA_DIR + row.masks
        image=cv2.imread(image_path)
        image= cv2.cvtColor(image,cv2.COLOR_BGR2RGB)
        mask= cv2.imread(mask_path, cv2.IMREAD_GRAYSCALE)
        mask= np.expand_dims(mask,axis=-1)

        if self.augmentations:
            data= self.augmentations(image=image,mask=mask)
            image= data['image']
            mask= data['mask']

        image= np.transpose(image,(2,0,1)).astype(np.float32)
        mask= np.transpose(mask,(2,0,1)).astype(np.float32)
        image= torch.Tensor(image) / 255.0
        mask= torch.round(torch.Tensor(mask) /255.0)

        return image,mask

trainset= SegmentationDataset(train_df,train_augs())
validset= SegmentationDataset(valid_df,valid_augs())

print(len(trainset))
print(len(validset))

idx=25

image,mask= trainset[idx]
helper.show_image(image,mask)
plt.show()

# Load the dataset into batches

train_loader= DataLoader(trainset,batch_size=BATCH_SIZE,num_workers=0,shuffle=True)
valid_loader= DataLoader(validset,batch_size=BATCH_SIZE)
## total no of batches in trainloader ####

print(len(train_loader))

print(len(valid_loader))

for i,m in train_loader:
    print(f"One batch image shape :{i.shape }")
    print(f"One batch mask shape :{m.shape}")
    break;

##### Creating Segmentation model ####################################
class SegmentationModel(nn.Module):
    def __init__(self):
        super(SegmentationModel,self).__init__()
        self.backbone= smp.Unet(encoder_name=ENCODER, encoder_weights=WEIGHTS, in_channels=3,classes=1,activation=None)

    def forward(self,images,masks=None):
        logits= self.backbone(images)
        if mask != None :
            return logits,DiceLoss(mode='binary')(logits,masks) + nn.BCEWithLogitsLoss()(logits,masks)
        else:
            return logits

model = SegmentationModel()
print(model)
model.to(DEVICE)

###### Creating train and valid function #####

def train_fn(dataloader,model,optimizer):
    ### set the model to training mode
    model.train()
    total_loss=0.0
    for images,masks in tqdm(dataloader):
        images=images.to(DEVICE)
        masks= masks.to(DEVICE)
        optimizer.zero_grad()
        logits,loss =model(images,masks)
        loss.backward()
        optimizer.step()
        total_loss+=loss.item()

    return total_loss/len(dataloader)

def valid_fn(dataloader,model):
    # set the model to evaluation mode
    model.eval()
    total_loss=0.0
    with torch.no_grad():
        for images,masks in tqdm(dataloader):
            images=  images.to(DEVICE)
            masks= masks.to(DEVICE)
            logits,loss= model(images,masks)
            total_loss+=loss.item()
        return total_loss/len(dataloader)

optimizer= torch.optim.Adam(model.parameters(),lr=LR)
best_loss= np.Inf

for i in range(EPOCHS):
    train_loss= train_fn(train_loader,model,optimizer)
    valid_loss= valid_fn(valid_loader,model)

    if valid_loss < best_loss:
        torch.save(model.state_dict(),'road_dataset/best_model.pth')
        print("saved model")
        best_loss= valid_loss

    print(f"Epoch:{i+1} Train Loss:{train_loss} Valid_loss :{valid_loss}")

idx=30
model.load_state_dict(torch.load('road_dataset/best_model.pth'))
image,mask= validset[idx]
# adding extra dimension for the batch
logits_mask= model(image.to(DEVICE).unsqueeze(0))
pred_mask = torch.sigmoid(logits_mask)
pred_mask= (pred_mask>0.5) *1.0

helper.show_image(image,mask,pred_mask.detach().cpu().squeeze(0))
plt.show()


import re
import pandas as pd
from sklearn.externals import joblib

def clean_html(raw_html):
  cleanr = re.compile('<.*?>')
  cleantext = re.sub(cleanr, '', raw_html)
  return cleantext

def extract_sku_text_features(train_file, test_file, feat_train_file, feat_test_file,
                                analyzer='char', ngram_range=(1, 4), lowercase=True, stem=False, stop_words=None):
    feat_names = []
    df = pd.read_csv(train_file, header=None, names=['country','sku_id','title','category_lvl_1','category_lvl_2','category_lvl_3','short_description','price','product_type'])
    df1 = pd.read_csv(test_file, header=None, names=['country','sku_id','title','category_lvl_1','category_lvl_2','category_lvl_3','short_description','price','product_type'])

    n_trains = df.shape[0]

    df = pd.concat((df,df1))

    df['sku1'] = df['sku_id'].map(lambda x: x[:5])
    df['sku2'] = df['sku_id'].map(lambda x: x[5:9])

    feat= pd.crosstab(index=df['category_lvl_1'], columns=df['sku1'])
    feat.columns = ['cat1_%s' % x for x in feat.columns]
    feat = feat.reset_index()
    df = df.merge(feat, how='left', on='category_lvl_1')
    feat_names.extend(feat.columns[1:].tolist())

    feat= pd.crosstab(index=df['category_lvl_2'], columns=df['sku1'])
    feat.columns = ['cat2_%s' % x for x in feat.columns]
    feat = feat.reset_index()
    df = df.merge(feat, how='left', on='category_lvl_2')
    feat_names.extend(feat.columns[1:].tolist())

    feat= pd.crosstab(index=df['category_lvl_3'], columns=df['sku1'])
    feat.columns = ['cat3_%s' % x for x in feat.columns]
    feat = feat.reset_index()
    df = df.merge(feat, how='left', on='category_lvl_3')
    feat_names.extend(feat.columns[1:].tolist())

    X = df[feat_names].astype(float).values

    X_tr = X[:n_trains,:]
    X_val = X[n_trains:,:]

    print('saving %s' % feat_train_file)
    joblib.dump(X_tr, feat_train_file)

    print('saving %s' % feat_test_file)
    joblib.dump(X_val, feat_test_file)

if __name__ == "__main__":
    # Data loading

    train_file = '../data/data_train.csv'
    test_file = '../data/data_valid.csv'

    feat_train_file = '../features/sku_feat.trn'
    feat_test_file = '../features/sku_feat.tst'
    extract_sku_text_features(train_file, test_file, feat_train_file, feat_test_file)


import sys
import os
import yaml
import json
from pickle import load, dump
from tqdm import tqdm

from skopt import BayesSearchCV
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.metrics import f1_score
from xgboost import XGBClassifier

from src.models_training.utils import select_candidate, tqdm_skopt

def prepare_data(candidate, data, features):
    candidate_name = candidate['training_field']
    candidate_feature = candidate['candidate']
    X_train, y_train = data['X_train'], data['y_train']
    X_train, y_train, _ = select_candidate(candidate_name, candidate_feature, features, X_train, y_train)

    return X_train, y_train

if __name__ == "__main__":

    if len(sys.argv) != 5:
        sys.stderr.write("Arguments error. Usage:\n")
        sys.stderr.write("\tpython train.py features-dir-path scheme-path model-dir-path param_name\n")
        sys.exit(1)

    train_input = os.path.join(sys.argv[1], "data.pickle")
    pipe_file = os.path.join(sys.argv[1], "pipe.pickle")
    scheme_file = sys.argv[2]

    os.makedirs(os.path.join(sys.argv[3]), exist_ok=True)

    # load data scheme and features
    with open(train_input, 'rb') as f1:
        data = load(f1)
    with open(scheme_file, 'rb') as f_s:
        scheme = json.load(f_s)
    with open(pipe_file, 'rb') as f1:
        pipe_feature = load(f1)
    features = pipe_feature.get_feature_names()

    params = yaml.safe_load(open("params.yaml"))[sys.argv[4]]

    optimize = params['optimize']
    n_iter = params['n_iter']
    n_estimators = params['n_estimators']

    if not optimize:
        learning_rate = params['learning_rate']
        max_depth = params['max_depth']
        max_leaf_nodes = params['max_leaf_nodes']

    n_scheme = len(scheme.keys())
    n_cpu = os.cpu_count()
    n_cv = 3  # cross validation for optimization
    n_points = max(n_cpu // n_cv, 1)

    for candidate, candidate_name in tqdm(zip(scheme.values(), scheme.keys())):

        X_train, y_train = prepare_data(candidate, data, features)
        # pipe_feature_post = Pipeline([
        #     ('power_transformer', PowerTransformer()),
        # ])
        #
        # X_train = pipe_feature_post.fit_transform(X_train)
        model = {}
        # model['pipe_feature_post'] = pipe_feature_post

        if optimize:
            model_output = os.path.join(sys.argv[3], candidate_name + '-' + "optimized_GB")
            pipe = BayesSearchCV(
                GradientBoostingClassifier(n_estimators=n_estimators, verbose=0),
                {
                    'max_leaf_nodes': (10, 60),
                    'learning_rate': (0.05, 0.2, 'uniform'),
                    'max_depth': (3, 100),  # integer valued parameter
                },
                n_points=n_points,
                n_jobs=n_cv * n_points,
                n_iter=n_iter,
                cv=n_cv,
                verbose=0,
                scoring='f1_macro',
            )
            pipe.fit(X_train, y_train)

            print("val. score: %s" % pipe.best_score_)
            model['model'] = pipe

        else:
            model_output = os.path.join(sys.argv[3], candidate_name + '-' + "GB")
            gbc = XGBClassifier(learning_rate=learning_rate, n_estimators=n_estimators,
                                max_depth=max_depth, verbosity=1, n_jobs=n_cpu,
                                eval_metric=f1_score)
            gbc.fit(X_train, y_train)
            model['model'] = gbc

        with open(model_output, 'wb') as f1:
            dump(model, f1)


import torch
import torch.nn as nn
import torch.utils.data as data
import numpy as np
import os

from utils.data import GestureData
from models.res2net import res2net50

def test():
    os.environ["CUDA_VISIBLE_DEVICES"] = "1,0"
    path = "/home/dshah/I6_Gestures"
    saved_model_path = "/home/dshah/2dgesturenet/res2net50.pth"

    test_data = GestureData(path, mode="test")
    test_loader = data.DataLoader(test_data, batch_size=16, shuffle=True, num_workers=4)

    model = nn.DataParallel(res2net50(num_classes=35))
    model = model.cuda()
    model.load_state_dict(torch.load(saved_model_path))

    model.eval()
    test_acc = []
    log_nth = 10
    for i, (inputs, targets) in enumerate(test_loader):
        inputs, targets = inputs.cuda().to(torch.float), targets.cuda().to(torch.long)
        outputs = model(inputs)

        _, preds = torch.max(outputs, 1)

        acc = np.mean((targets == preds).detach().cpu().numpy())
        test_acc.append(acc)

        if i % log_nth == 0:
            avg_acc = sum(test_acc)/len(test_acc)
            print("Average accuracy: %.3f" % avg_acc)

    mean_test_acc = sum(test_acc)/len(test_acc)
    print("Mean accuracy over full test set: %.3f" % mean_test_acc)

if __name__ == "__main__":
    test()


"""
Torch dataset object for synthetically rendered spatial data.
"""

import os
import torch
from scipy.io.wavfile import read
import os
import glob

def get_dataset(dir):
    original_files = glob.glob(os.path.join(dir, "*_original.wav"))
    converted_files = []
    for original_file in original_files:
        converted_file = original_file.replace(
            "_original.wav", "_converted.wav")
        converted_files.append(converted_file)
    return original_files, converted_files

def load_wav(full_path):
    sampling_rate, data = read(full_path)
    return data, sampling_rate

class LLVCDataset(torch.utils.data.Dataset):

    def __init__(
        self,
        dir,
        sr,
        wav_len,
        dset
    ):
        assert dset in [
            "train",
            "val",
            "dev"
        ], "`dset` must be one of ['train', 'val', 'dev']"
        self.dset = dset
        file_dir = os.path.join(dir, dset)
        self.wav_len = wav_len
        self.sr = sr
        self.original_files, self.converted_files = get_dataset(
            file_dir
        )

    def __len__(self):
        return len(self.original_files)

    def __getitem__(self, idx):
        original_wav = self.original_files[idx]
        converted_wav = self.converted_files[idx]

        original_data, o_sr = load_wav(original_wav)
        converted_data, c_sr = load_wav(converted_wav)

        assert o_sr == self.sr, f"Expected {self.sr}Hz, got {o_sr}Hz for file {original_wav}"
        assert c_sr == self.sr, f"Expected {self.sr}Hz, got {c_sr}Hz for file {converted_wav}"

        converted = torch.from_numpy(original_data)
        gt = torch.from_numpy(converted_data)

        converted = converted.unsqueeze(0).to(torch.float) / 32768
        gt = gt.unsqueeze(0).to(torch.float) / 32768

        if gt.shape[-1] < self.wav_len:
            gt = torch.cat(
                (gt, torch.zeros(1, self.wav_len - gt.shape[-1])), dim=1)
        else:
            gt = gt[:, : self.wav_len]

        if converted.shape[-1] < self.wav_len:
            converted = torch.cat(
                (converted, torch.zeros(1, self.wav_len - converted.shape[-1])), dim=1
            )
        else:
            converted = converted[:, : self.wav_len]

        return converted, gt


import numpy as np
import tensorflow as tf
from bench.core.executer import Executer

def bbox_overlaps(bboxes1, bboxes2, mode='iou', is_aligned=False, eps=1e-6):
    """Calculate overlap between two set of bboxes.

    If ``is_aligned`` is ``False``, then calculate the ious between each bbox
    of bboxes1 and bboxes2, otherwise the ious between each aligned pair of
    bboxes1 and bboxes2.

    Args:
        bboxes1 (Tensor): shape (m, 4) in <x1, y1, x2, y2> format or empty.
        bboxes2 (Tensor): shape (n, 4) in <x1, y1, x2, y2> format or empty.
            If is_aligned is ``True``, then m and n must be equal.
        mode (str): "iou" (intersection over union) or iof (intersection over
            foreground).

    Returns:
        ious(Tensor): shape (m, n) if is_aligned == False else shape (m, 1)

    Example:
        >>> bboxes1 = torch.FloatTensor([
        >>>     [0, 0, 10, 10],
        >>>     [10, 10, 20, 20],
        >>>     [32, 32, 38, 42],
        >>> ])
        >>> bboxes2 = torch.FloatTensor([
        >>>     [0, 0, 10, 20],
        >>>     [0, 10, 10, 19],
        >>>     [10, 10, 20, 20],
        >>> ])
        >>> bbox_overlaps(bboxes1, bboxes2)
        tensor([[0.5000, 0.0000, 0.0000],
                [0.0000, 0.0000, 1.0000],
                [0.0000, 0.0000, 0.0000]])

    Example:
        >>> empty = torch.FloatTensor([])
        >>> nonempty = torch.FloatTensor([
        >>>     [0, 0, 10, 9],
        >>> ])
        >>> assert tuple(bbox_overlaps(empty, nonempty).shape) == (0, 1)
        >>> assert tuple(bbox_overlaps(nonempty, empty).shape) == (1, 0)
        >>> assert tuple(bbox_overlaps(empty, empty).shape) == (0, 0)
    """

    assert mode in ['giou', 'iou', 'iof']
    # Either the boxes are empty or the length of boxes's last dimenstion is 4
    assert (bboxes1.shape[-1] == 4 or bboxes1.shape[0] == 0)
    assert (bboxes2.shape[-1] == 4 or bboxes2.shape[0] == 0)

    # Batch dim must be the same
    # Batch dim: (B1, B2, ... Bn)
    assert bboxes1.shape[:-2] == bboxes2.shape[:-2]
    batch_shape = bboxes1.shape[:-2]

    rows = bboxes1.shape[0]
    cols = bboxes2.shape[0]
    if is_aligned:
        assert rows == cols

    if rows * cols == 0:
        return tf.zeros([rows, 1]) if is_aligned else tf.zeros([rows, cols])

    area1 = (bboxes1[:, 2] - bboxes1[:, 0]) * (
        bboxes1[:, 3] - bboxes1[:, 1])
    area2 = (bboxes2[:, 2] - bboxes2[:, 0]) * (
        bboxes2[:, 3] - bboxes2[:, 1])

    if is_aligned:
        lt = tf.math.maximum(bboxes1[:, :2], bboxes2[:, :2])  # [rows, 2]
        rb = tf.math.minimum(bboxes1[:, 2:], bboxes2[:, 2:])  # [rows, 2]

        wh = tf.clip_by_value((rb - lt), clip_value_min=0, clip_value_max=tf.float32.max)  # [rows, 2]
        overlap = wh[:, 0] * wh[:, 1]

        if mode == 'iou':
            union = area1 + area2 - overlap
        else:
            union = area1
        if mode == 'giou':
            enclosed_lt = tf.math.minimum(bboxes1[:, :2], bboxes2[:, :2])
            enclosed_rb = tf.math.maximum(bboxes1[:, 2:], bboxes2[:, 2:])
    else:
        lt = tf.math.maximum(bboxes1[:, None, :2], bboxes2[:, :2])  # [rows, cols, 2]
        rb = tf.math.minimum(bboxes1[:, None, 2:], bboxes2[:, 2:])  # [rows, cols, 2]

        wh = tf.clip_by_value((rb - lt), clip_value_min=0, clip_value_max=tf.float32.max)  # [rows, cols, 2]
        overlap = wh[:, :, 0] * wh[:, :, 1]

        if mode == 'iou' or mode == 'giou':
            union = area1[:, None] + area2 - overlap
        else:
            union = area1[:, None]
        if mode == 'giou':
            enclosed_lt = tf.math.minimum(bboxes1[:, :, None, :2],
                                    bboxes2[:, None, :, :2])
            enclosed_rb = tf.math.maximum(bboxes1[:, :, None, 2:],
                                    bboxes2[:, None, :, 2:])

    eps = tf.constant([eps])
    union = tf.math.maximum(union, eps)
    ious = overlap / union

    if mode in ['iou', 'iof']:
        return ious
    # calculate gious
    enclose_wh = tf.clip_by_value((enclosed_rb - enclosed_lt), clip_value_min=0, clip_value_max=tf.float32.max)
    enclose_area = enclose_wh[:, 0] * enclose_wh[:, 1]
    enclose_area = tf.math.maximum(enclose_area, eps)
    gious = ious - (enclose_area - union) / enclose_area
    return gious

def giou_loss(pred, target, eps=1e-6):
    """IoU loss.

    Computing the IoU loss between a set of predicted bboxes and target bboxes.
    The loss is calculated as negative log of IoU.

    Args:
        pred (torch.Tensor): Predicted bboxes of format (x1, y1, x2, y2),
            shape (n, 4).
        target (torch.Tensor): Corresponding gt bboxes, shape (n, 4).
        eps (float): Eps to avoid log(0).

    Return:
        torch.Tensor: Loss tensor.
    """
    gious = bbox_overlaps(pred, target, mode='giou', is_aligned=True, eps=eps)
    loss = 1 - gious
    return loss

def args_adaptor(np_args):
    pred = tf.convert_to_tensor(np_args[0], tf.float32)
    # pred.requires_grad = True
    target = tf.convert_to_tensor(np_args[1], tf.float32)

    return [pred, target]

def executer_creator():
    return Executer(giou_loss, args_adaptor)


from sqlalchemy import (
    MetaData, Table, Column,
    Integer, DateTime
)

meta = MetaData()

activation = Table(
    'activation', meta,

    Column('id', Integer, primary_key=True),
    Column('is_activated', Integer, nullable=False),
    Column('pub_date', DateTime, nullable=False)
)


from django.shortcuts import render, redirect
from .forms import SuCreationForm
from .models import su_additional
from django.contrib.auth.models import User,Group
from django.db import IntegrityError
import uuid as UUID
# Create your views here.

def su_register(request):
    if request.method == "POST":
        print(request.content_params)
        print(request.POST)
        form = SuCreationForm(request.POST)
        if form.is_valid():
            form.save()
            su_username = form.cleaned_data.get('username')
            group = Group.objects.create(name=su_username)
            user = User.objects.get_by_natural_key(username=su_username)
            user.groups.add(group)
            su_setting = su_additional(user=user, uuid_for_reg=UUID.uuid4())
            su_setting.save()
            return redirect('customers')
    form = SuCreationForm()
    return render(request, 'customer/register.html', {'form': form})


from flask import Flask, render_template, url_for, request
from Parser import Parser_PDDL

app = Flask(__name__)

@app.route('/')
def index():
    return render_template('index.html')

@app.route('/uploader', methods=['GET', 'POST'])
def upload_file():
    if request.method == 'POST':
        domain = request.files['Domain_file']
        problem = request.files['Problem_file']
        domain.save('PDDLFiles/domain.pddl')
        problem.save('PDDLFiles/problem.pddl')
        parser = Parser_PDDL('PDDLFiles/domain.pddl', 'PDDLFiles/problem.pddl')
        model1 = parser.parse_PDDL()
        print(model1)
        return 'Success'

if __name__ == "__main__":
    app.run(debug=True)


"""

--- Main fulltext processing pipeline ---

Be aware: Running the whole pipeline can take 16++ hours, depending on number of processors assigned, setting 5+
processors is beneficial

General prerequisites:
    Python >= 3.5
    NLTK == 3.2.2
    Treetagger == 3.2 Linux
    Treetaggerwrapper == 2.2.2
    NLTK datasets (nltk.download()) and specified TreeTagger directory
    (following instructions at http://www.cis.uni-muenchen.de/~schmid/tools/TreeTagger/)
    NB: Treetagger 3.2 on Windows is currently not supported due to version differences with the Linux version
    of Treetagger 3.2

Processing steps:
    1) Download PMC Open Access Subset: https://www.ncbi.nlm.nih.gov/pmc/tools/openftlist/
    2) Either download abstracts separately for the journals to be analyzed (using get_pubmeddata from
       dataminingfunctions) or extract them from the fulltexts directly
    3) Put fulltext textfiles into subfolders for each journal in ../data/fulltexts/*** (where *** are the ftJournals
       names below) and the abstracts file ("searchresults") into ../data/fulltexts/abstracts/*** with
       subfolders "/id_article/abstracttext_pubdate_year_pmid_articletitle_journal_title_keyword_doi/searchresults"
       (where *** are the abstractsJournals names specified below)
    4) Make sure that provided doi -> pmid matching .csvs are in basePath/ids/ (default ../data/fulltexts/ids/) to
       reproduce our analysis, otherwise ID matching can also be done by calling the Pubmed API, check convert_id.py
    5) Set numProc parameter to number of processors to be assigned to the task (default is 4,
       but 5+ is recommended if possible)

Output:
    .json and .csv dataframes in statsDir (default output path is "../data/fulltexts/stats/")

"""

from multiprocessing import Pool
from functools import partial
import os
import sys
sys.path.append('../')

from readability_fulltexts_vs_abstracts import parse_fulltexts as pf
from readability_fulltexts_vs_abstracts import analyze_fulltexts as af
from readability_fulltexts_vs_abstracts import combine_abstracts_fulltexts as caf

# Initialize variables

# Number of processors for multithreading
numProc = 4

basePath = '../data/fulltexts/'
outPath = basePath + 'output/'
outPathONE = outPath + 'PLoS_ONE/'
abstractsPath = basePath + 'abstracts/'
statsDir = basePath + 'stats/'
idPath = basePath + 'ids/'

ftJournals = ['BMC_Biol', 'eLife', 'Genome_Biol', 'PLoS_Med', 'PLoS_Biol']
abstractJournalsOthers = ['bmc_biol[journal]', 'elife[journal]', 'genome_biol[journal]',
                          'plos_med[journal]', 'plos_biol[journal]']
abstractJournalONE = 'plos_one[journal]'

# For id conversion at Pubmed (not necessary with defaults if doi -> pmid csv mappings exist)
email = 'placeholder'
tool = 'id_converter'

if not os.path.exists(outPath + 'metrics/'):
    os.makedirs(outPath + 'metrics/')

if not os.path.exists(outPath + 'pmid/'):
    os.makedirs(outPath + 'pmid/')

if not os.path.exists(outPathONE + 'pmid/'):
    os.makedirs(outPathONE + 'pmid/')

if not os.path.exists(idPath):
    os.makedirs(idPath)

if not os.path.exists(outPathONE + 'metrics/' + 'PLoS_ONE/'):
    os.makedirs(outPathONE + 'metrics/' + 'PLoS_ONE/')

if not os.path.exists(statsDir):
    os.makedirs(statsDir)

if __name__ == '__main__':

    # Parse raw text files
    pf.parse_bmc(basePath, outPath)
    pf.parse_elife(basePath, outPath)
    pf.parse_genbiol(basePath, outPath)
    pf.parse_plos(basePath, outPath)
    pf.parse_plosone(basePath, outPathONE)

    # Make list of parsed jsons from PLoS ONE
    ftJournalsONE = [t[:-5] for t in os.listdir(outPathONE) if t.endswith('.json')]

    # Other journals convert_id
    af.convert_id_others(outPath, idPath, ftJournals)

    # Other journals abstracts analysis
    pool = Pool(processes=numProc)  # Create multiprocessing Pool
    partialAbstracts = partial(af.analyze_abstracts_others, abstractsPath)  # Partial needed for additional arguments
    pool.map(partialAbstracts, abstractJournalsOthers)  # Map jsons to workers
    pool.close()

    # Other journals fulltexts analysis
    pool = Pool(processes=numProc)  # Create multiprocessing Pool
    partialFulltexts = partial(af.analyze_ft_others, outPath)  # Partial needed for additional arguments
    pool.map(partialFulltexts, ftJournals)  # Map jsons to workers
    pool.close()

    # PLoS ONE convert_id
    af.convert_id_plosone(outPathONE, idPath, ftJournalsONE)

    # PLoS ONE abstracts analysis
    af.analyze_abstracts_plosone(abstractsPath, abstractJournalONE)

    # PLoS ONE fulltexts analysis
    pool = Pool(processes=numProc)  # Create multiprocessing Pool
    partialFTONE = partial(af.analyze_ft_plosone, outPathONE)  # Partial is needed for additional arguments
    pool.map(partialFTONE, ftJournalsONE)  # Map jsons to workers
    pool.close()

    # Combine abstracts and fulltexts into a single dataframe
    caf.combine_abstracts_fulltexts_one(abstractsPath, outPathONE, statsDir, abstractJournalONE, ftJournalsONE)
    caf.combine_abstracts_fulltexts_others(abstractsPath, outPath, statsDir, abstractJournalsOthers, ftJournals)
    caf.combine_others_one(statsDir)


import unittest
import pytest
import logging, os
from lxml.etree import DTD, RelaxNG, parse, DocumentInvalid
import sldr.UnicodeSets as usets
from unicodedata import normalize
import re

@pytest.fixture(scope="session")
def validator(request):
    # return RelaxNG(file=os.path.join(os.path.dirname(__file__), '..', 'doc', 'sil.rng'))
    return DTD(file=os.path.join(os.path.dirname(__file__), '..', 'auxdata', 'sil.dtd'))

def iscldr(ldml):
    i = ldml.ldml.root.find(".//identity/special/sil:identity", {v:k for k,v in ldml.ldml.namespaces.items()})
    if i is not None and i.get('source', "") == "cldr":
        return True
    return False

def test_validate(ldml, validator, fixdata):
    """ Test LDML file conforms to sil/ldml rng """
    xml = parse(ldml.path)
    try:
        validator.assertValid(xml)
    except DocumentInvalid as e:
        # import pdb; pdb.set_trace()
        if fixdata and str(e).startswith("Did not expect element"):
            tag = str(e)[23:]
            tag = tag[:tag.find(" ")]
            if tag in ("generation", "language"):
                ldml.ldml.ensure_path('identity/version[@number="0.0.1"]', before=tag)
                ldml.dirty = True
                return
        assert False, str(e)

def test_exemplars(ldml):
    """ Test for overlaps between exemplars. Test that index chars are all in main exemplar """
    if iscldr(ldml):    # short circuit CLDR for now until they/we resolve the faults in their data
        return
    filename = os.path.basename(ldml.ldml.fname)    # get filename for reference
    exemplars = {}
    for e in ldml.ldml.root.findall('.//characters/exemplarCharacters'):
        t = e.get('type', None)
        s = usets.parse(e.text or "", 'NFD')
        if not len(s):
            continue
        exemplars[t] = s[0]
    if None not in exemplars:
        return
    else:
        main = exemplars[None]
    for k, v in exemplars.items():
        if k in (None, "index", "numbers"):
            continue
        diff = main & v
        assert not len(diff), filename + " Overlap found between main and %s" % (k)
    if 'index' in exemplars:
        if 'auxiliary' in exemplars:
            test = main.union(exemplars['auxiliary'])
        else:
            test = main
        m = set([x.lower() for x in exemplars['index']])
        diff = m - test
        # The following lines (down to the assert) added to handle digraphs whose components are present
        digraphcheck = True
        for digraph in diff:
            for c in digraph:
                if c not in test:
                    digraphcheck = False
                    break
        assert digraphcheck or not len(diff), filename + " Not all index entries found in main or auxiliary"

# THE WHOLE NEXT TEST IS EMILY CODE and is therefore probably JANKY
def test_syntax(ldml):
    """ Test for syntax errors in exemplars """
    if iscldr(ldml):    # short circuit CLDR for now until they/we resolve the faults in their data
        return
    filename = os.path.basename(ldml.ldml.fname)    # get filename for reference
    exemplars = {}
    exemplars_raw = {}
    exemplars_rawnocurly = {}
    for e in ldml.ldml.root.findall('.//characters/exemplarCharacters'):
        t = e.get('type', None)
        n = t or "main"
        exemplars_rawnocurly[t] = e.text[1:-1].strip().replace("\\", " \\").replace("{", " ").replace("}", " ").replace("  ", " ").split(' ') # adapted from the "get index exemplar" section of test_collation.py
        exemplars_raw[t] = e.text[1:-1].strip().split(' ') # adapted from the "get index exemplar" section of test_collation.py
        rawstring = e.text[1:-1].strip().replace(" ", "") # adapted from the "get index exemplar" section of test_collation.py
        s = usets.parse(e.text or "", 'NFD')
        if not len(s):
            continue
        exemplars[t] = s[0]
        # The following lines test if unicode hex values in all exemplars are properly formatted
        for i in exemplars_rawnocurly[t]:
            if "\\" in i:
                if r"\u" in i:
                    assert len(i)==6, filename + " " + n + " exemplar has unicode codepoint(s) missing hex digits: " + i
                if r"\U" in i:
                    assert len(i)==10, filename + " " + n + " exemplar has unicode codepoint(s) missing hex digits: " + i
                #this next assert does assume that spaces were added between units in an exemplar, since exemplars_rawnocurly can only insert a space BEFORE a backslash. So far nothing fails incorrectly because of that
                assert len(i)<3 or len(i)==6 or len(i)==10, filename + " " + n + " exemplar has unicode codepoint(s) missing 'u' or 'U': " + i
        # The following lines are a test if characters are incorrectly unescaped.
        # The problem with these coming tests is that if there are ranges that use special characters intentionally, they'll ping as errors.
        # However we can't solely test for "is it a valid regex" bc they might make a valid regex on accident.
        # Also it seems rare to use special characters intentionally in punctuation and numbers, so if this false error does happen, it'll be very uncommon.
        if 'punctuation' in exemplars:
            for p in exemplars_raw['punctuation']:
                for a in (("-", "hyphen"), (":", "colon"), ("&", "ampersand"), ("[", "square bracket"), ("]", "square bracket"), ("{", "curly bracket"), ("}", "curly bracket"), ("$", "dollar sign")):
                    if a[0] in p:
                        assert "\\"+a[0] in p, filename + " Unescaped " + a[1] + " in punctuation exemplar"
                #not sure how to test for a non-escaped backslash since that's used intentionally EVERYWHERE.
        if 'numbers' in exemplars:
            for n in exemplars_raw['numbers']:
                for b in (("-", "hyphen"), (":", "colon")):
                    if b[0] in n:
                        assert "\\"+b[0] in n, filename + " Unescaped " + b[1] + " in numbers exemplar"
        # there are probably more but I can't think of them atm
        # Assert below tests that exemplars are valid regular expressions: it's a catch-all for anything the tests above might miss
        assert _test_re("\"\"[" + rawstring + "]\"\""), filename + " " + n + " exemplar isn't a valid regex"

def test_diacritics (ldml):
    """Tests if the exemplars have diacritics listed separately rather than in their composed form. Only tests Latn diacritics atm"""
#    if iscldr(ldml):    # short circuit CLDR for now until they/we resolve the faults in their data
#        return
    filename = os.path.basename(ldml.ldml.fname)    # get filename for reference
    comb_diacritics = ["\u0300","\u0301","\u0302","\u0303","\u0304","\u0305","\u0306","\u0307",
                  "\u0308","\u0309","\u030A","\u030B","\u030C","\u030D","\u030E","\u030F",
                  "\u0310","\u0311","\u0312","\u0313","\u0314","\u0315","\u0316","\u0317",
                  "\u0318","\u0319","\u031A","\u031B","\u031C","\u031D","\u031E","\u031F",
                  "\u0320","\u0321","\u0322","\u0323","\u0324","\u0325","\u0326","\u0327",
                  "\u0328","\u0329","\u032A","\u032B","\u032C","\u032D","\u032E","\u032F",
                  "\u0330","\u0331","\u0332","\u0333","\u0334","\u0335","\u0336","\u0337",
                  "\u0338","\u0339","\u033A","\u033B","\u033C","\u033D","\u033E","\u033F",
                  "\u1AB0","\u1AB1","\u1AB2","\u1AB3","\u1AB4","\u1AB5","\u1AB6","\u1AB7",
                  "\u1AB8","\u1AB9","\u1ABA","\u1ABB","\u1ABC","\u1ABD","\u1ABE","\u1ABF",
                  "\u1AC0","\u1AC1","\u1AC2","\u1AC3","\u1AC4","\u1AC5","\u1AC6","\u1AC7",
                  "\u1AC8","\u1AC9","\u1ACA","\u1ACB","\u1ACC","\u1ACD","\u1ACE",
                  "\u1DC0","\u1DC1","\u1DC2","\u1DC3","\u1DC4","\u1DC5","\u1DC6","\u1DC7",
                  "\u1DC8","\u1DC9","\u1DCA","\u1DCB","\u1DCC","\u1DCD","\u1DCE","\u1DCF",
                  "\u1DD0","\u1DD1","\u1DD2","\u1DD3","\u1DD4","\u1DD5","\u1DD6","\u1DD7",
                  "\u1DD8","\u1DD9","\u1DDA","\u1DDB","\u1DDC","\u1DDD","\u1DDE","\u1DDF",
                  "\u1DE0","\u1DE1","\u1DE2","\u1DE3","\u1DE4","\u1DE5","\u1DE6","\u1DE7",
                  "\u1DE8","\u1DE9","\u1DEA","\u1DEB","\u1DEC","\u1DED","\u1DEE","\u1DEF",
                  "\u1DF0","\u1DF1","\u1DF2","\u1DF3","\u1DF4","\u1DF5","\u1DF6","\u1DF7",
                  "\u1DF8","\u1DF9","\u1DFA","\u1DFB","\u1DFC","\u1DFD","\u1DFE","\u1DFF"]
    exemplars = {}
    for e in ldml.ldml.root.findall('.//characters/exemplarCharacters'):
        t = e.get('type', None)
        n = t or "main"
        print (t)
        print(n)
        print(e.text)
        if n == "punctuation" or n == "numbers":
            continue
        if not len(usets.parse(e.text, 'NFC')):
            continue
        exemplars[n] = usets.parse(e.text, 'NFC')[0].asSet()
        print(exemplars[n])
        for c in exemplars[n]:
            print (c)
            assert c not in comb_diacritics, filename + " diacritics aren't composed"

def _duplicate_test(base, ldml, path=""):
    filename = os.path.basename(ldml.fname)    # get filename for reference
    idents = set()
    for c in base:
        if c.tag in ("variable",):
            continue
        ident = [c.tag]
        ident.extend(["{}={}".format(k, v) for k, v in sorted(c.attrib.items())])
        #for n in list(ldml.keyContexts.get(c.tag, ldml.keys)) + ['draft']:
        #    v = c.get(n, None)
        #    if v is not None:
        #        ident.append("{}={}".format(n, v))
        i = ":".join(ident)
        if len(ident) > 1:
            assert i not in idents, filename + " Found overlapping elements for " + path + "/" + i
        idents.add(i)
        if len(c) and _duplicate_test(c, ldml, path=path + "/" + i):
            return True
    return False

def test_duplicates(ldml):
    """ Test that no two elements have the same identifying feature values """
    _duplicate_test(ldml.ldml.root, ldml.ldml)

def _test_re(string):
    """ Return True if string is a valid regular expression; False otherwise """
    try:
        x = re.compile(string)
        return True
    except re.error:
        return False