from django.db import models, transaction
from rest_framework.authtoken.models import Token

from .element_types import INPUT, DATETIME, SELECT, RADIO, CHECKBOX, DATE, TIME, INT, FLOAT, TEXTAREA, BOOLEAN, \
    FILE_INPUT
from django.contrib.auth.models import User

from .sub_form_fields import get_related_attrs

class UserProfile(models.Model):
    """User profile"""
    user = models.OneToOneField(User, related_name="user_profile", on_delete=models.CASCADE)

    access_level = models.PositiveBigIntegerField(default=0)

    @property
    def token(self) -> str:
        token, created = Token.objects.get_or_create(user=self.user)
        return str(token.key)

    def __str__(self):
        return str(self.user)

class Template(models.Model):
    """Base template for a form that can be filled later"""
    creator = models.ForeignKey(UserProfile, related_name="templates",
                                on_delete=models.CASCADE)
    title = models.CharField(max_length=255, default="")

    access_level = models.PositiveBigIntegerField(default=0)

    is_paginated = models.BooleanField(default=False)

    @property
    def forms_count(self):
        return self.forms.count()

    def __str__(self):
        return str(self.title)

class Form(models.Model):
    """A Form is like a fork of a template, it includes elements,
    but elements are regarded as answers to the base template"""

    filler = models.ForeignKey(UserProfile, related_name="filled_forms", on_delete=models.CASCADE)
    fork_date = models.DateTimeField(auto_now_add=True)
    last_change_date = models.DateTimeField(auto_now=True)
    template = models.ForeignKey(Template, related_name="forms", on_delete=models.CASCADE)
    description = models.CharField(max_length=255, default="")

    class Meta:
        unique_together = ['template', 'description']

    def __str__(self):
        return "%s - %s" % (str(self.template), str(self.description))

class SubForm(models.Model):
    """
    SubForms make up different sections of each form
    """
    title = models.CharField(max_length=255, blank=True, null=True)
    description = models.CharField(max_length=1000, blank=True, null=True)
    order = models.IntegerField(default=0)
    template = models.ForeignKey(Template, related_name="sub_forms", on_delete=models.CASCADE)

    condition_element_type = models.CharField(max_length=255, blank=True, null=True)
    condition_element_pk = models.IntegerField(blank=True, null=True)
    condition_element_value = models.CharField(max_length=255, blank=True, null=True)

    def __str__(self):
        return "%s - %s" % (str(self.template), str(self.title))

class Field(models.Model):
    """
    A field consists of one or more field elements
    """
    sub_form = models.ForeignKey(SubForm, related_name="fields", on_delete=models.CASCADE)
    title = models.CharField(max_length=255, blank=True, null=True)
    order = models.IntegerField(default=0)
    is_grid = models.BooleanField(default=False)

    condition_element_type = models.CharField(max_length=255, blank=True, null=True)
    condition_element_pk = models.IntegerField(blank=True, null=True)
    condition_element_value = models.CharField(max_length=255, blank=True, null=True)

    def __str__(self):
        return "%s - %s" % (str(self.sub_form), str(self.title))

class Element(models.Model):
    """
    Base class for an input element (defaults to simple text field)
    value and filters fields may be overridden
    to cope with other input types

    # this field should be set according to the data type
    # Ex.: a simple text filed may be stored in CharField
    # Ex.: to store a datetime, value may be DateTimeField
    value = models.CharField(max_length=1024, blank=True, null=True)

    # array of available filters on an elements
    filters = ['icontains', 'startswith', 'endswith']

    all sub classes of element will be related to a sub form with a related named of the
    following format: fields_class_name, where class_name is the name of the subclass :)

    """

    # if the given element is used as answer
    # title, type, order, and field will be inherited from the question field
    # when serializing the element
    title = models.CharField(max_length=255)

    # Ex.: Input, checkbox, datetime, ...
    type = "ABC"

    # show this element based on some condition
    condition_element_type = models.CharField(max_length=255, blank=True, null=True)
    condition_element_pk = models.IntegerField(blank=True, null=True)
    condition_element_value = models.CharField(max_length=255, blank=True, null=True)

    # some element may have more than one value at a given time (Ex. Check box)
    # these elements save their values in "values" field as a CharField model
    # default value field for elements is "value"
    value_field = 'value'

    # any extra data that may be needed for the given element, can be saved in Data as
    # display, value pairs
    data = models.ManyToManyField("Data", blank=True)

    # display order of the field
    order = models.IntegerField(default=0)
    field = models.ForeignKey(Field, related_name="elements_%(class)s",
                              on_delete=models.CASCADE, blank=True, null=True)

    # these fields are used for an element that
    # will be regarded as an answer to an element of the same type
    answer_of = models.ForeignKey("self", related_name="answers",
                                  on_delete=models.CASCADE, blank=True, null=True)
    form = models.ForeignKey(Form, related_name="answers_%(class)s",
                             on_delete=models.CASCADE, blank=True, null=True)

    # indicates if element is editable
    disabled = models.BooleanField(default=False)

    quantitative_filters = [{"value": '', "display": "مساوی"},
                            {"value": 'gt', "display": "بزرگتر"},
                            {"value": 'lt', "display": "کوچکتر"},
                            {"value": 'gte', "display": "بزرگتر مساوی"},
                            {"value": 'lte', "display": "کوچکتر مساوی"}
                            ]
    literal_filters = [
        {"value": "", "display": "مساوی"},
        {"value": 'icontains', "display": "دربر دارد"},
        {"value": 'startswith', "display": "شروع میشود با"},
        {"value": 'endswith', "display": "پایان میابد با"}
    ]

    @classmethod
    def related_name_to_form(cls):
        return "answers_%s" % str.lower(cls.__name__)

    def short_str(self, data):
        return data[:50] + (data[50:] and ' ...')

    @property
    def display_title(self):
        return "%s -> %s -> %s" % (self.short_str(self.field.sub_form.title) if self.field.sub_form.title else "",
                                   self.short_str(self.field.title) if self.field.title else " ",
                                   self.short_str(self.title) if self.title else "")

    @property
    def display_title_full(self):
        return "%s -> %s -> %s" % (self.field.sub_form.title if self.field.sub_form.title else "",
                                   self.field.title if self.field.title else " ",
                                   self.title if self.title else "")

    @property
    def uid(self):
        return str(self.type) + str(self.pk)

    def __str__(self):
        return "%s - %s" % (str(self.field), str(self.title))

    class Meta:
        unique_together = ['answer_of', 'form']
        abstract = True

class Input(Element):
    """ Simple Text Input """
    value = models.CharField(max_length=1024, blank=True, null=True)
    type = INPUT
    filters = Element.literal_filters

class FileInput(Element):
    """ Simple Text Input """
    value = models.FileField(blank=True, null=True)
    type = FILE_INPUT
    filters = []

class Boolean(Element):
    """ Simple Text Input """
    value = models.BooleanField(blank=True, null=True)
    type = BOOLEAN
    filters = [{"value": "", "display": "مساوی"}]

class TextArea(Element):
    """ Simple Text Input """
    value = models.CharField(max_length=10240, blank=True, null=True)
    type = TEXTAREA
    filters = Element.literal_filters

class DateTimeElement(Element):
    """Date time field """

    value = models.DateTimeField(blank=True, null=True)
    type = DATETIME
    filters = Element.quantitative_filters

class SelectElement(Element):
    """Html select element with options"""

    value = models.CharField(max_length=1024, blank=True, null=True)
    type = SELECT
    filters = [{"value": '', "display": 'مساوی'}, ]  # empty filter string means exact match

class RadioElement(Element):
    """Html radio element with options"""

    value = models.CharField(max_length=1024, blank=True, null=True)
    type = RADIO
    filters = [{"value": '', "display": 'مساوی'}, ]  # empty filter string means exact match

class CharField(models.Model):
    """Raw char field"""
    value = models.CharField(max_length=1024, blank=True, null=True)

class CheckboxElement(Element):
    """Html checkbox element with options"""

    values = models.ManyToManyField(CharField, blank=True)
    type = CHECKBOX
    filters = [{"value": 'value__contains', "display": 'شامل'}, ]  # empty filter string means exact match
    value_field = 'values'

class DateElement(Element):
    """Html date element with options"""

    value = models.DateField(blank=True, null=True)
    type = DATE
    filters = Element.quantitative_filters

class TimeElement(Element):
    """Html TimeField element with options"""

    value = models.TimeField(blank=True, null=True)
    type = TIME
    filters = Element.quantitative_filters

class IntegerField(Element):
    """Html Int element with options"""

    value = models.IntegerField(blank=True, null=True)
    type = INT
    filters = Element.quantitative_filters

class FloatField(Element):
    """Html TimeField element with options"""

    value = models.FloatField(blank=True, null=True)
    type = FLOAT
    filters = Element.quantitative_filters

class Data(models.Model):
    """ Extra data used on select, radio, checkbox elements"""
    value = models.CharField(max_length=255)
    display = models.CharField(max_length=255)

elements = {
    INPUT: Input,
    DATE: DateElement,
    TIME: TimeElement,
    DATETIME: DateTimeElement,
    RADIO: RadioElement,
    CHECKBOX: CheckboxElement,
    SELECT: SelectElement,
    INT: IntegerField,
    TEXTAREA: TextArea,
    FLOAT: FloatField,
    BOOLEAN: Boolean,
    FILE_INPUT: FileInput
}


#!/usr/bin/env python3

from wsgiref.simple_server import make_server

import falcon

class ThingsResource:
    def on_get(self, req, resp):
        """Handles GET requests"""
        resp.status = falcon.HTTP_200
        resp.content_type = falcon.MEDIA_TEXT  # Default is JSON, so override
        resp.text = ('\nTwo things awe me most, the starry sky above me and the moral law within me.\n'
                     '    ~ Immanuel Kant\n')

# A CORS policy is required for <umarcor.github.io/hwstudio> to work with a backend served at <127.0.0.1>.
# - https://github.com/godotengine/godot/pull/40542
# - https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS
# - https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS/Errors/CORSMissingAllowOrigin
# Enable a simple CORS policy for all responses.
# - https://falcon.readthedocs.io/en/3.0.1/api/cors.html
app = falcon.App(cors_enable=True)

app.add_route('/things', ThingsResource())

if __name__ == '__main__':
    with make_server('', 8000, app) as httpd:
        print('Serving on port 8000...')
        httpd.serve_forever()


import pandas as pd
import numpy.random as nr
from sklearn.model_selection import GridSearchCV

from sklearn.model_selection import train_test_split
from sklearn import metrics
import numpy as np

import math

from sklearn.preprocessing import StandardScaler
from sklearn.metrics import accuracy_score, f1_score

from sklearn import ensemble

from sklearn import linear_model

from sklearn.preprocessing import LabelEncoder

#funkcija koja traži najbolje hiperparametre gradient boostinga
def gbHyperparametars(X_train, y_train):
    algorithmParameters = dict() #dict hiperparametara koje ćemo testirati

    #popunjavanje algorithmParameters
    nEstimatorsValues = []
    for i in range(6, 12):
        nEstimatorsValues.append(2**i)

    algorithmParameters['n_estimators'] = nEstimatorsValues

    learningRateValues  = []
    for i in range(3):
        learningRateValues.append(0.01 * 10**i)

    algorithmParameters['learning_rate'] = learningRateValues

    algorithmParameters['max_depth'] = [2, 3, 4]

    model = ensemble.GradientBoostingClassifier()

    #traženje i ispis najboljih hiperparametara
    grid = GridSearchCV(model, algorithmParameters)
    grid.fit(X_train, y_train)

    print(grid.best_estimator_.n_estimators)
    print(grid.best_estimator_.max_depth)
    print(grid.best_estimator_.learning_rate)

    classifierGB = ensemble.GradientBoostingClassifier(n_estimators = grid.best_estimator_.n_estimators, max_depth = grid.best_estimator_.max_depth,
                                                learning_rate = grid.best_estimator_.learning_rate)

    return classifierGB

#funkcija koja traži najbolje hiperparametre random forest
def rfHyperparametars(X_train, y_train, n_estimators):

    algorithmParameters = dict()

    algorithmParameters['min_samples_split'] = [2, 3, 4, 8]

    algorithmParameters['max_features'] = [0.01, 0.05, 0.1]

    #algorithmParameters['max_depth'] = [16, 32, None]

    model = ensemble.RandomForestClassifier(n_estimators = n_estimators)

    grid = GridSearchCV(model, algorithmParameters)
    grid.fit(X_train, y_train)

    #print(grid.best_estimator_.max_depth)
    print(grid.best_estimator_.min_samples_split)
    print(grid.best_estimator_.max_features)

    classifierRF = ensemble.RandomForestClassifier(n_estimators = n_estimators, min_samples_split = grid.best_estimator_.min_samples_split,
                                                    max_depth = None, max_features = grid.best_estimator_.max_features)

    return classifierRF

#hiperparametri za logističku regresiju
def lrHyperparametars(X_train, y_train):

    algorithmParameters = dict()

    algorithmParameters['penalty'] = ['l1', 'l2']

    CValues   = []
    for i in range(1, 15):
        CValues.append(0.001 * 2**i)

    algorithmParameters['C'] = CValues

    #algorithmParameters['dual'] = [True, False]

    model = linear_model.LogisticRegression()

    grid = GridSearchCV(model, algorithmParameters)
    grid.fit(X_train, y_train)

    print(grid.best_estimator_.penalty)
    #print(grid.best_estimator_.dual)
    print(grid.best_estimator_.C)

    classifierLR = linear_model.LogisticRegression(penalty = grid.best_estimator_.penalty, C = grid.best_estimator_.C)
    #dual = grid.best_estimator_.dual)

    return classifierLR

def getUserDescription(df_train, df_test, featureName = 'description'):

    userTrain = df_train.user

    userTextTrain = []

    for dictionary in userTrain:

        temp = ''

        for key, value in dictionary.items():

            if key == featureName:
                temp = value

        userTextTrain.append(temp)

    userTextTrain = np.asarray(userTextTrain)

    userTest = df_test.user

    userTextTest = []

    for dictionary in userTest:

        temp = ''

        for key, value in dictionary.items():

            if key == featureName:
                temp = value

        userTextTest.append(temp)

    userTextTest = np.asarray(userTextTest)

    return userTextTrain, userTextTest

def getHashtag(df_train, df_test):

    firstEntities = df_train.entities

    hashtagsArray = []

    for dictionary in firstEntities:
        temp = ''

        for key, values in dictionary.items():

            if key == 'hashtags':

                for i in range(values.__len__()):

                    if isinstance(values[i], dict):
                        for keyTemp, valueTemp in values[i].items():
                            if keyTemp == 'text':

                                temp = valueTemp

        hashtagsArray.append(temp)

    hashtagsArray = np.asarray(hashtagsArray)

    firstEntities = df_test.entities

    hashtagsArrayTest = []

    for dictionary in firstEntities:
        temp = ''

        for key, values in dictionary.items():

            if key == 'hashtags':

                for i in range(values.__len__()):

                    if isinstance(values[i], dict):
                        for keyTemp, valueTemp in values[i].items():
                            if keyTemp == 'text':

                                temp = valueTemp

        hashtagsArrayTest.append(temp)

    hashtagsArrayTest = np.asarray(hashtagsArrayTest)

    return hashtagsArray, hashtagsArrayTest

def create_output_file(filename, predictions, sentences, labels = []):

    if labels.__len__() == 0:
        df = pd.DataFrame({'Tekst' : sentences, 'Predikcija' : predictions})
    else:
        df = pd.DataFrame({'Tekst' : sentences, 'Predikcija' : predictions, 'Ispravne kategorije' : labels})

    df.to_csv(filename + '.csv')

df1 = pd.read_json('C:\\Users\\viktor\\Downloads\\hs_data\\share\\sexism.json', lines=True)
df2 = pd.read_json('C:\\Users\\viktor\\Downloads\\hs_data\\share\\racism.json', lines=True)
df3 = pd.read_json('C:\\Users\\viktor\\Downloads\\hs_data\\share\\neither.json', lines=True)

frames = [df1, df2, df3]

print(df1.__len__())
print(df2.__len__())
print(df3.__len__())

df = pd.concat(frames, ignore_index = True)

print(df.__len__())

from sklearn.utils import shuffle
df = shuffle(df)

del df['contributors']
del df['coordinates']
del df['extended_entities']
del df['geo']
del df['quoted_status']
del df['is_quote_status']
del df['quoted_status_id']
del df['quoted_status_id_str']
del df['retweeted']
del df['favorited']
del df['truncated']
del df['withheld_in_countries']
del df['possibly_sensitive']
del df['possibly_sensitive_appealable']
del df['id_str']
del df['in_reply_to_status_id']
del df['in_reply_to_screen_name']
del df['in_reply_to_status_id_str']
del df['in_reply_to_user_id']
del df['in_reply_to_user_id_str']

#df['Annotation'] = df['Annotation'].astype('category').cat.codes
df['lang'] = df['lang'].astype('category').cat.codes

from sklearn.feature_extraction.text import TfidfVectorizer
from nltk.corpus import stopwords

sw = stopwords.words("english")
vectorizer = TfidfVectorizer(lowercase=True, stop_words = sw, binary = True, sublinear_tf  = True, norm = None)

df_train, df_test = train_test_split(df, test_size = 0.2)

#tweetovi i label
X_train = df_train['text'].as_matrix()
y_train = df_train['Annotation'].as_matrix()

X_test = df_test['text'].as_matrix()
y_test = df_test['Annotation'].as_matrix()

X_train = vectorizer.fit_transform(X_train).toarray()
X_test = vectorizer.transform(X_test).toarray()

#dodaj user description
userTextTrain, userTextTest = getUserDescription(df_train, df_test)

userTextTrain = vectorizer.fit_transform(userTextTrain).toarray()
userTextTest = vectorizer.transform(userTextTest).toarray()

X_train = np.concatenate((X_train, userTextTrain), axis=1)
X_test = np.concatenate((X_test, userTextTest), axis=1)

#dodaj favorite_count
favorite_countTrain = df_train['favorite_count'].as_matrix()
favorite_countTest = df_test['favorite_count'].as_matrix()

ss = StandardScaler()

favorite_countTrain = ss.fit_transform(favorite_countTrain)
favorite_countTest = ss.transform(favorite_countTest)

favorite_countTrain = np.reshape(favorite_countTrain, (-1, 1))
favorite_countTest = np.reshape(favorite_countTest, (-1, 1))

X_train = np.concatenate((X_train, favorite_countTrain), axis=1)
X_test = np.concatenate((X_test, favorite_countTest), axis=1)

#dodaj retweet_count
retweet_countTrain = df_train['retweet_count'].as_matrix()
retweet_countTest = df_test['retweet_count'].as_matrix()

retweet_countTrain = ss.fit_transform(retweet_countTrain)
retweet_countTest = ss.transform(retweet_countTest)

retweet_countTrain = np.reshape(retweet_countTrain, (-1, 1))
retweet_countTest = np.reshape(retweet_countTest, (-1, 1))

X_train = np.concatenate((X_train, retweet_countTrain), axis=1)
X_test = np.concatenate((X_test, retweet_countTest), axis=1)

mlb = LabelEncoder()
y_train = mlb.fit_transform(y_train)
y_test = mlb.transform(y_test)

#dodaj user description
userTextTrain, userTextTest = getUserDescription(df_train, df_test, 'name')

userTextTrain = vectorizer.fit_transform(userTextTrain).toarray()
userTextTest = vectorizer.transform(userTextTest).toarray()

X_train = np.concatenate((X_train, userTextTrain), axis=1)
X_test = np.concatenate((X_test, userTextTest), axis=1)

#dodaj hashtags

hashTrain, hashTest = getHashtag(df_train, df_test)

hashTrain = vectorizer.fit_transform(hashTrain).toarray()
hashTest = vectorizer.transform(hashTest).toarray()

X_train = np.concatenate((X_train, hashTrain), axis=1)
X_test = np.concatenate((X_test, hashTest), axis=1)

#isprobaj svm
from sklearn.svm import LinearSVC

svc = LinearSVC(C = 0.0016)

svc.fit(X_train, y_train)

pred = svc.predict(X_test)

print(pred)
print(accuracy_score(pred, y_test))
print(f1_score(pred, y_test, average = 'macro'))

'''
#isprobaj lr i odredi joj hiperparametre
#lgr = lrHyperparametars(X_train, y_train)
lgr = linear_model.LogisticRegression(C = 0.128, penalty = 'l1')
lgr.fit(X_train, y_train)
pred = lgr.predict(X_test)
print(pred)
print(accuracy_score(pred, y_test))
print(f1_score(pred, y_test, average = 'macro'))
'''

#isprobaj rf i odredi mu hiperparametre
#rf = rfHyperparametars(X_train, y_train, 15) #None 0.1
rf = ensemble.RandomForestClassifier(n_estimators = 25, criterion="entropy", min_samples_split = 4, max_features = 0.05, n_jobs = -1)

rf.fit(X_train, y_train)
pred = rf.predict(X_test)
print(pred)
print(accuracy_score(pred, y_test))
print(f1_score(pred, y_test, average = 'macro'))

'''

#isprobaj gradient boosting
gb = ensemble.GradientBoostingClassifier()

gb.fit(X_train, y_train)
pred = gb.predict(X_test)
print(pred)
print(accuracy_score(pred, y_test))
print(f1_score(pred, y_test, average = 'micro'))
'''

pred = mlb.inverse_transform(pred)

create_output_file("testAll", pred, df_test['text'].as_matrix(), mlb.inverse_transform(y_test))


from django.urls import path

from authors.apps.articles import views

app_name = "articles"

urlpatterns = [
    path('articles/', views.ArticleViewSet.as_view(
        {'get': 'list', "post": "create"}), name='articles-all'),
    path('articles/<pk>/', views.ArticleViewSet.as_view(
        {"get": "retrieve", "put": "update", "patch": "partial_update",
         "delete": "destroy"}), name='single-article'),
    path('rate/<slug>/', views.RatingAPIView.as_view(), name='rating'),
    path('articles/<pk>/like/', views.LikeViewSet.as_view(
        {"patch": "partial_update"}), name='like_article'),
    path('articles/<pk>/dislike/', views.DisLikeViewSet.as_view(
        {"patch": "partial_update"}), name='dislike_article'),
    path('articles/<pk>/comments/', views.CommentViewSet.as_view(
        {'get': 'list', "post": "create"}), name='comments-all'),
    path('articles/<pk>/comments/<id>/', views.CommentViewSet.as_view(
        {'get': 'retrieve', "put": "update",
         "delete": "destroy", "post": "create_reply"}), name='single-comment'),
    path('favorites/', views.FavoriteViewSet.as_view({"get": "list"}),
         name='favorites'),
    path('articles/<slug>/favorite',
         views.FavoriteViewSet.as_view({
             "put": "update", "delete": "destroy"}), name="favorite_article"),
]


from typing import List

from volatility.framework import renderers, constants, interfaces
from volatility.framework import exceptions, contexts
from volatility.framework.automagic import linux
from volatility.framework.configuration import requirements
from volatility.framework.interfaces import plugins
from volatility.framework.objects import utility
from volatility.framework.renderers import format_hints

class Lsmod(plugins.PluginInterface):
    """Lists loaded kernel modules"""

    @classmethod
    def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]:
        return [
            requirements.TranslationLayerRequirement(
                name = 'primary', description = 'Memory layer for the kernel', architectures = ["Intel32", "Intel64"]),
            requirements.SymbolTableRequirement(name = "vmlinux", description = "Linux kernel symbols")
        ]

    @classmethod
    def list_modules(cls, context: interfaces.context.ContextInterface, layer_name: str, vmlinux_symbols: str):
        """Lists all the modules in the primary layer"""
        linux.LinuxUtilities.aslr_mask_symbol_table(context, vmlinux_symbols, layer_name)

        vmlinux = contexts.Module(context, vmlinux_symbols, layer_name, 0, absolute_symbol_addresses = True)

        modules = vmlinux.object(symbol_name = "modules").cast("list_head")

        table_name = modules.vol.type_name.split(constants.BANG)[0]

        for module in modules.to_list(table_name + constants.BANG + "module", "list"):
            yield module

    def _generator(self):
        for module in self.list_modules(self.context, self.config['primary'], self.config['vmlinux']):

            mod_size = module.get_init_size() + module.get_core_size()

            mod_name = utility.array_to_string(module.name)

            yield 0, (format_hints.Hex(module.vol.offset), mod_name, mod_size)

    def run(self):
        return renderers.TreeGrid([("Offset", format_hints.Hex), ("Name", str), ("Size", int)], self._generator())


#!/usr/bin/env python
# coding: utf-8

# ### **METADATA CREATION AND OBJECT CONCATENATION**
# #### Script takes the output files from STARsolo and gets them ready for the downstream analysis

import sys, os
import numpy as np
import pandas as pd
pd.options.mode.chained_assignment = None
import scipy as sc
from scipy.io import mmread, mminfo, mmwrite
from scipy import sparse
from scipy.sparse import vstack
from os import listdir
from os.path import isfile, join

#setting working directory
align_results_dir = './Results/STARsolo_Output/'

# #creation of metadata containing cell number, barcode, stage and sample
# stages = next(os.walk(align_results_dir))[1]
# for stage in stages:
#     samples = next(os.walk(align_results_dir + stage + '/'))[1]
#     for sample in samples:
#         lanes = next(os.walk(align_results_dir + stage + '/' + sample + '/'))[1]
#         for lane in lanes:
#             batches = next(os.walk(align_results_dir + stage + '/' + sample + '/' + lane + '/'))[1]
#             batches = [k for k in batches if 'Solo.out' in k]
#             for batch in batches:
#                 features = next(os.walk(align_results_dir + stage + '/' + sample + '/' + lane + '/' + batch))[1]
#                 for feature in features:
#                     if os.path.isfile(align_results_dir + stage + '/' + sample + '/' + lane + '/' + batch + '/' + feature + '/raw/barcodes.tsv'):
#                         bc = pd.read_csv(align_results_dir + stage + '/' + sample + '/' + lane + '/' + batch + '/' + feature + '/raw/barcodes.tsv', header=None)
#                         bc.rename(columns={0: 'barcode'}, inplace=True)
#                         df = bc.drop_duplicates(subset=['barcode'], inplace=False)
#                         df['cell'] = 'cell_' + df.reset_index().index.astype(str)
#                         df['sample'] = sample
#                         df['stage'] = stage
#                         df['stage'] = df['stage'].str.replace('_', '')
#                         df = pd.DataFrame(df)
#                         df.to_csv(align_results_dir + stage + '/' + sample + '/' + lane + '/' + batch + '/' + feature + '/raw/meta.tab', sep="\t", index=False)
#                     else:
#                         print(sample + '_' + lane + ' not found')
#                         break

# # deleting Gene Expression column from features.tsv and metadata concatenation of all samples
# # just on Genefull, as Velocyto has 3 different matrices
# genes = []
# metadata = []
# features = ['GeneFull']
# for stage in stages:
#     samples = next(os.walk(align_results_dir + stage + '/'))[1]
#     for sample in samples:
#         lanes = next(os.walk(align_results_dir + stage + '/' + sample + '/'))[1]
#         for lane in lanes:
#             batches = next(os.walk(align_results_dir + stage + '/' + sample + '/' + lane + '/'))[1]
#             batches = [k for k in batches if 'Solo.out' in k]
#             for batch in batches:
#                 for feature in features:
#                     if os.path.isfile(align_results_dir + stage + '/' + sample + '/' + lane + '/' + batch + '/' + feature + '/raw/barcodes.tsv'):
#                         #Load the data
#                         gene = pd.read_csv(align_results_dir + stage + '/' + sample + '/' + lane + '/' + batch + '/' + feature + '/raw/features.tsv',sep="\t",skiprows=0, header=None)
#                         meta = pd.read_csv(align_results_dir + stage + '/' + sample + '/' + lane + '/' + batch + '/' + feature + '/raw/meta.tab',sep="\t",skiprows=0)
#                         gene = gene.drop(gene.columns[2], axis=1)
#                         metadata.append(meta)
#                     else:
#                         print(sample + '_' + lane + ' file not found')

# df_metadata = pd.concat(metadata, ignore_index=True)

# gene.to_csv(align_results_dir + 'features.tsv', sep='\t', index=False)
# df_metadata.to_csv(align_results_dir + 'metadata_whole.tab', sep='\t', index=False)
# print('Gene Features and Metadata file saved')

# creation of whole matrix object concatenation all samples (high-resource process)
whole_matrix = []
features = ['GeneFull']
stages = next(os.walk(align_results_dir))[1]
for stage in stages:
    samples = next(os.walk(align_results_dir + stage + '/'))[1]
    for sample in samples:
        lanes = next(os.walk(align_results_dir + stage + '/' + sample + '/'))[1]
        for lane in lanes:
            batches = next(os.walk(align_results_dir + stage + '/' + sample + '/' + lane + '/'))[1]
            batches = [k for k in batches if 'Solo.out' in k]
            for batch in batches:
                for feature in features:
                    if os.path.isfile(align_results_dir + stage + '/' + sample + '/' + lane + '/' + batch + '/' + feature + '/raw/barcodes.tsv'):
                        #Load the data
                        mat = mmread(align_results_dir + stage + '/' + sample + '/' + lane + '/' + batch + '/' + feature + '/' + 'raw/matrix.mtx').tocsr().transpose()
                        whole_matrix.append(mat)
                        break
                    else:
                        print(sample + '_' + lane + ' file not found')

whole_matrix = sc.sparse.vstack((whole_matrix))
mmwrite(align_results_dir + 'matrix_whole', whole_matrix)

print('Whole Matrix file saved')


import sys
import socket
import functools

import easycli as cli

from .constants import IGMP_PORT, IGMP_ADDRESS, VERBS, DEFAULT_DBFILE, \
    DEFAULT_TIMEOUT, BINARY_CONTENTTYPES
from . import protocol, cache, resolve, http

hostname_arg = functools.partial(cli.Argument, 'hostname')
nocache_arg = functools.partial(cli.Argument, '--nocache', action='store_true')
address_arg = functools.partial(cli.Argument, '-a', '--address')
port_arg = functools.partial(cli.Argument, '-p', '--port', type=int)
short_arg = functools.partial(
    cli.Argument,
    '-s',
    '--short',
    action='store_true'
)
timeout_arg = functools.partial(
    cli.Argument,
    '-t',
    '--timeout',
    type=int,
    default=DEFAULT_TIMEOUT,
    help=f'Seconds wait before exit, 0: infinite, default: {DEFAULT_TIMEOUT}'
)
noresolve_arg = functools.partial(
    cli.Argument,
    '--noresolve',
    action='store_true',
    help='Do not resolve the name over network.'
)
forceresolve_arg = functools.partial(
    cli.Argument,
    '-f', '--forceresolve',
    action='store_true',
    help='Force to resolve the name over network and update cache.'
)

stdout = sys.stdout
stderr = sys.stderr

def output(*a, **kw):
    print(*a, file=stdout, **kw)

def error(*a, **kw):
    print(*a, file=stderr, **kw)

def printrecord(name, addr, cache, short=False):
    if short:
        output(addr)
        return

    flags = ' [cache]' if cache else ''
    output(f'{addr} {name}{flags}')

class Answer(cli.SubCommand):
    """Answer command line interface."""

    __command__ = 'answer'
    __aliases__ = ['a', 'ans']
    __arguments__ = [
        hostname_arg(default=IGMP_ADDRESS),
        address_arg(default=IGMP_ADDRESS, help=f'Default: {IGMP_PORT}'),
        port_arg(default=IGMP_PORT, help=f'Default: {IGMP_ADDRESS}'),
    ]

    def __call__(self, args):
        output(f'Answering {args.hostname} to {args.address}:{args.port}')
        protocol.answer(args.hostname, address=args.address, port=args.port)

class Sniff(cli.SubCommand):
    """Sniff IGMP packets."""

    __command__ = 'sniff'
    __aliases__ = ['s']
    __arguments__ = [
    ]

    def __call__(self, args):
        output(f'Listening to {IGMP_ADDRESS}:{IGMP_PORT}')
        try:
            for verb, name, addr, port in protocol.sniff():
                output(f'{addr}:{port} {VERBS.get(verb)} {name}')
        except KeyboardInterrupt:
            error('Terminated by user.')
            return 3

class Find(cli.SubCommand):
    """Find an ip address by it's name."""

    __command__ = 'find'
    __aliases__ = ['f']
    __arguments__ = [
        cli.Argument('pattern'),
        nocache_arg(),
        short_arg(),
        timeout_arg(),
    ]

    def __call__(self, args):
        if not args.nocache:
            with cache.DB(args.dbfile) as db:
                for name, addr in db.find(args.pattern):
                    printrecord(name, addr, True, short=args.short)

        # Searching network
        for n, a in protocol.find(args.pattern, timeout=args.timeout):
            printrecord(n, a, False, short=args.short)

class Resolve(cli.SubCommand):
    """Resolve an ip address by it's name."""

    __command__ = 'resolve'
    __aliases__ = ['r', 'd']
    __arguments__ = [
        hostname_arg(),
        cli.Mutex(
            noresolve_arg(),
            forceresolve_arg(),
        ),
        short_arg(),
        timeout_arg(),
    ]

    def __call__(self, args):
        addr, cached = resolve(
            args.hostname,
            args.timeout,
            args.forceresolve,
            args.noresolve,
            args.dbfile
        )
        printrecord(args.hostname, addr, cached, short=args.short)

class HTTP(cli.SubCommand):
    """Send HTTP request to a UNS host."""

    __command__ = 'http'
    __aliases__ = ['h']
    __arguments__ = [
        cli.Mutex(
            noresolve_arg(),
            forceresolve_arg(),
        ),
        timeout_arg(),
        cli.Argument(
            '-H', '--header',
            action='append',
            default=[],
            help='Extra  header to include in the request when sending HTTP '
                 'to a server.'
        ),
        cli.Argument(
            '-i', '--include-headers',
            action='store_true',
            help='Include protocol response headers in the output.'
        ),
        cli.Argument(
            '-b', '--binary-output',
            action='store_true',
            help='Treat response as binary.'
        ),
        port_arg(),
        cli.Argument('verb'),
        cli.Argument('url'),
        cli.Argument(
            'fields',
            metavar='[?|@]NAME=VALUE',
            default=[],
            nargs='*',
            help='You may use ":" prefix to send a file as request body '
                 'for example: "uns http put hostname :path/to/file".'
        )
    ]

    def __call__(self, args):
        query = []
        fields = []
        files = []
        body = ''
        path_ = ''
        port = ''

        urlparts = args.url.split('/')
        addr, _ = resolve(
            urlparts[0],
            args.timeout,
            args.forceresolve,
            args.noresolve,
            args.dbfile
        )
        if len(urlparts) > 1:
            path_ = '/' + '/'.join(urlparts[1:])

        if args.port:
            port = f':{args.port}'

        # Fields
        for f in args.fields:
            if '=' not in f:
                body = ' '.join(args.fields)
                break

            k, v = f.split('=')
            if k[0] == '?':
                query.append((k[1:], v))
            elif k[0] == '@':
                files.append((k[1:], open(v, mode='rb')))
            else:
                fields.append((k, v))

        # Open body if it's a filename
        if body and body.startswith(':'):
            body = open(body[1:], mode='rb')

        # Request headers
        reqheaders = {
            k.strip(): v.strip() for k, v in
            [h.split(':') for h in args.header]
        }
        resp = http.request(
            args.verb.upper(),
            f'http://{addr}{port}{path_}',
            query=query,
            form=body if body else fields,
            files=files,
            headers=reqheaders
        )

        # Response headers
        respheaders = [
            f'{resp.status_code} {resp.reason} '
            f'HTTP/{resp.raw.version / 10:.1f}'
        ]
        respheaders += [f'{k}: {v}' for k, v in resp.headers.items()]
        if args.include_headers:
            output('\n'.join(respheaders), end='\n\n')

        # Handle exception
        if resp.status_code >= 400:
            error(resp.text, end='')
            return 1

        # Binary output
        binary = False
        if args.binary_output:
            binary = True

        elif resp.headers.get('content-type'):
            for c in BINARY_CONTENTTYPES:
                if c.match(resp.headers['content-type']):
                    binary = True
                    break

        if binary:
            stdout.buffer.write(resp.content)
        else:
            output(resp.text, end='')

class UNS(cli.Root):
    """UNS root command line handler."""

    __completion__ = True
    __help__ = 'UNS utility.'
    __arguments__ = [
        cli.Argument('-v', '--version', action='store_true'),
        cli.Argument(
            '--dbfile',
            default=DEFAULT_DBFILE,
            help=f'Database file, default: {DEFAULT_DBFILE}'
        ),
        Resolve,
        Answer,
        Sniff,
        Find,
        HTTP,
    ]

    def main(self, *a, **k):
        try:
            return super().main(*a, **k)
        except socket.timeout:
            error('Timeout reached.')
            return 2

        except KeyboardInterrupt:
            error('Terminated by user.')
            return 3

        except cache.InvalidDBFileError as ex:
            error(f'Invalid input file: {ex}')
            return 4

        except cache.HostNotFoundError as ex:
            error(f'Cannot find: {ex}.')
            return 5

    def __call__(self, args):
        if args.version:
            import uns
            output(uns.__version__)
            return

        self._parser.print_help(file=stdout)


import os.path
import time
import openai
from pathlib import Path
import urllib.parse
import json
import re
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import random
from bs4 import BeautifulSoup
from datetime import datetime

def filter_html(html_string):
    soup = BeautifulSoup(html_string, "html.parser")

    # Remove all script and style elements
    for script in soup(["script", "style"]):
        script.decompose()

    # Remove the div with id 'coverage'
    coverage_div = soup.find(id="coverage")
    if coverage_div:
        coverage_div.decompose()

    # Convert HTML object back to a string without additional newlines
    text = str(soup)

    return text

def format_past_actions(past_actions):
    if not past_actions:
        return "No actions available. "

    formatted_actions = "\n"
    for action in past_actions:
        formatted_actions += f"Step {action['step']}: Action: {action['action']} | Observation: {action['observation']} | Coverage Percentage: {action['coverage percentage']}%\n"
    return formatted_actions

def custom_logger(msg, log_messages):
    print(msg)
    log_messages.append(f"{datetime.now()}:{msg}")
    return log_messages

def run_ui_test(url, delay, interactions, load_wait_time, test_type, output_dir):
    # Check if the given URL is a local file path
    if os.path.isfile(url):
        # If it is a local file, convert the file path to a proper URL
        url = Path(url).as_uri()
    elif not urllib.parse.urlsplit(url).scheme:
        # If it's a relative file path, convert it to an absolute path first, then to a proper URL
        url = Path(os.path.abspath(url)).as_uri()

    log_messages = []

    config = {
        "url": url,
        "delay": delay,
        "interactions": interactions,
        "load_wait_time": load_wait_time,
        "test_type": test_type,
        "output_dir": output_dir,
    }

    # Add configuration to log
    log_messages = custom_logger(
        f"Configuration: {json.dumps(config, indent=2)}", log_messages
    )

    log_messages = custom_logger(f"Starting the test on URL: {url}", log_messages)

    # OpenAI key loading would be required here
    with open("openai_key.json", "r") as file:
        openai.api_key = json.load(file)["key"]

    # Open the web browser and navigate to the app's URL
    browser = webdriver.Chrome()
    browser.get(url)

    # Wait for the elements to load
    WebDriverWait(browser, load_wait_time).until(
        EC.presence_of_element_located((By.TAG_NAME, "body"))
    )
    log_messages = custom_logger("Web page loaded successfully.", log_messages)

    # Get the filtered HTML source code of the page
    filtered_html = filter_html(browser.page_source)

    # Start testing
    past_actions = []
    for i in range(interactions):
        # Refresh the list of buttons before each interaction
        buttons = browser.find_elements(By.XPATH, "//button")
        display_element = browser.find_element(By.ID, "display")

        element = None
        if test_type == "monkey":
            # Choose a random button
            element = random.choice(buttons)
        elif test_type in ["gpt-4", "gpt-3.5-turbo"]:
            # Create list of clickable elements using IDs
            clickable_elements_data = [button.get_attribute("id") for button in buttons]

            # Create the prompt for the GPT model with task description
            prompt = (
                f"Given a web application, you are tasked with testing its functionality. "
                f"Here is the filtered HTML source code of the web application: '{filtered_html}'. "
                f"Here are the available interactable GUI elements: {clickable_elements_data}. "
                f"Here are the ordered past actions that you have done for this test (first element was the first action of the test and the last element was the previous action): {format_past_actions(past_actions)}"
                f"Please output the id of the element to click on next and provide a brief explanation or reasoning for your choice. "
                f"Remember, the goal is to test as many different features as possible to find potential bugs and make sure to include edge cases."
            )

            # Define the function for GPT
            functions = [
                {
                    "name": "select_element",
                    "description": "Selects an element given its ID and provides an explanation for the choice",
                    "parameters": {
                        "type": "object",
                        "properties": {
                            "id": {
                                "type": "string",
                                "description": "The id of the element to select",
                            },
                            "explanation": {
                                "type": "string",
                                "description": "The reasoning behind the selection of this element",
                            },
                        },
                        "required": ["id", "explanation"],
                    },
                }
            ]

            # Ask the GPT model for the next action
            response = openai.ChatCompletion.create(
                model=test_type,
                messages=[{"role": "user", "content": prompt}],
                functions=functions,
                function_call={"name": "select_element"},
            )

            response_message = response["choices"][0]["message"]

            # Check if GPT wanted to call a function
            if response_message.get("function_call"):
                function_args = json.loads(
                    response_message["function_call"]["arguments"]
                )
                action_id = function_args.get("id")
                log_messages = custom_logger(
                    function_args.get("explanation"), log_messages
                )

                for button in buttons:
                    if button.get_attribute("id") == action_id:
                        element = button
                        break
                if not element:
                    raise Exception(f"No button found with id: {action_id}")
            else:
                raise Exception(
                    f"The model did not make a function call in the response: {response_message}"
                )

        else:
            raise ValueError(f"Invalid test type: {test_type}")

        element.click()

        # Check for alert and accept it if present
        try:
            alert = browser.switch_to.alert
            log_messages = custom_logger(
                f"Alert found with message: {alert.text}. Accepting it.", log_messages
            )
            alert.accept()
        except Exception as e:
            pass  # no alert, so pass

        # Get coverage percentage
        try:
            coverage_element = browser.find_element(By.ID, "percentage")
            coverage_text = coverage_element.text
            coverage_percentage = re.search(r"(\d+.\d+)%", coverage_text).group(1)
        except Exception as e:
            log_messages = custom_logger(
                f"Could not find coverage element or extract percentage: {str(e)}",
                log_messages,
            )
            coverage_percentage = None

        # Record the observation after the action
        current_observation = display_element.get_attribute("value")
        current_action = element.get_attribute("id")

        log_messages = custom_logger(
            f"Action {i+1}: {test_type.capitalize()} tester clicking button with id: '{current_action}' | Current observation: {current_observation} | Coverage: {coverage_percentage}%",
            log_messages,
        )

        # Record action
        past_actions.append(
            {
                "step": (i + 1),
                "action": current_action,
                "observation": current_observation,
                "coverage percentage": coverage_percentage,
            }
        )

        time.sleep(delay)  # Wait a bit between actions for the page to update

    # Extract and log the code coverage data after all interactions
    coverage_json_string = browser.execute_script("return JSON.stringify(coverage)")
    coverage_data = json.loads(coverage_json_string)
    covered_blocks = sum(
        1
        for function in coverage_data.values()
        for block, times in function.items()
        if times > 0
    )
    total_blocks = sum(len(function) for function in coverage_data.values())
    coverage_percentage = (
        round((covered_blocks / total_blocks) * 100, 2) if total_blocks > 0 else 0
    )

    log_messages = custom_logger(
        f"Final coverage data: {json.dumps(coverage_data, indent=2)}", log_messages
    )

    log_messages = custom_logger(
        f"Detailed coverage calculation explanation: Out of the total number of {total_blocks} blocks across all functions, "
        f"{covered_blocks} were covered (i.e., executed at least once during the test). This leads to a final "
        f"coverage percentage of {coverage_percentage}%. This percentage represents the ratio of the number of "
        f"covered blocks to the total number of blocks, giving equal weight to each block.",
        log_messages,
    )

    # Close the driver
    log_messages = custom_logger("Test run completed.", log_messages)
    browser.quit()

    # Time-stamp to uniquely identify this run
    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")

    # Create a unique subfolder for each test run
    output_dir = os.path.join(output_dir, timestamp)

    # Create results directory if it doesn't exist
    os.makedirs(output_dir, exist_ok=True)

    # Save click arguments
    with open(os.path.join(output_dir, "config.json"), "w") as file:
        json.dump(config, file, indent=4)

    # Save past actions to an output file
    with open(os.path.join(output_dir, "past_actions.json"), "w") as file:
        json.dump(past_actions, file, indent=4)

    log_messages = custom_logger(
        f"Past actions saved to: {os.path.join(output_dir, 'past_actions.json')}",
        log_messages,
    )

    # Save log messages to a log file
    with open(os.path.join(output_dir, "output.log"), "w") as log_file:
        for message in log_messages:
            log_file.write(f"{message}\n")

    log_messages = custom_logger(
        f"Logs saved to: {os.path.join(output_dir, 'output.log')}", log_messages
    )

    return log_messages, past_actions


from __future__ import unicode_literals
from django.contrib.auth.models import User
from django.contrib.auth.validators import ASCIIUsernameValidator
from django.db import models

class AdminUser(User):
    username_validator = ASCIIUsernameValidator()

    class Meta:
        proxy = True
        verbose_name = 'Administrador'

class StaffUser(User):
    username_validator = ASCIIUsernameValidator()

    class Meta:
        proxy = True
        verbose_name = 'staff'
        permissions = (
            ('add_address', 'Can add address'),
            ('change_address', 'Can change address'),
            ('delete_address', 'Can delete address'),
        )

class CostomerServiceUser(User):
    username_validator = ASCIIUsernameValidator()

    class Meta:
        proxy = True
        verbose_name = 'Servicio al cliente'

class ClientUser(User):
    username_validator = ASCIIUsernameValidator()

    class Meta:
        proxy = True
        verbose_name = 'Cliente'
        permissions = (
            ('view_clientuser', 'can see client user'),
        )

class Address(models.Model):
    user = models.ForeignKey(
        ClientUser,
        verbose_name="Cliente",
        related_name="client_relationship_address",
        related_query_name="client_relationship_address",
        on_delete=models.CASCADE
    )
    state = models.CharField(
        verbose_name="Estado",
        max_length=20,
        blank=False,
        null=False
    )
    municipality = models.CharField(
        verbose_name="Municipio/delegación",
        max_length=50,
        blank=False,
        null=False
    )
    suburb = models.CharField(
        verbose_name="Colonia",
        max_length=50,
        blank=False,
        null=False
    )
    street = models.CharField(
        verbose_name="Calle",
        max_length=70,
        blank=False,
        null=False
    )
    outdoor_number = models.IntegerField(
        verbose_name="Numero exterior",
        null=False,
        blank=False,
        default=0
    )
    zipcode = models.IntegerField(
        verbose_name="codigo postal",
        null=False,
        blank=False,
        default=0
    )

    class Meta:
        permissions = (
            ('view_address', 'can see address'),
        )


from store_product.models import Store_product_document
from django.conf import settings
from util import couch_db_util

def exe(
     id
    ,store_id
    ,product_id
    ,name
    ,price
    ,value_customer_price
    ,crv
    ,is_taxable
    ,is_sale_report
    ,p_type
    ,p_tag
    ,sku_lst
    ,cost
    ,vendor
    ,buydown
    ,breakdown_assoc_lst
):
    store_product_document = Store_product_document(
         id = id
        ,product_id  = product_id
        ,store_id = store_id
        ,d_type = settings.STORE_PRODUCT_DOCUMENT_TYPE
        ,name = name
        ,price = price
        ,value_customer_price = value_customer_price
        ,crv = crv
        ,is_taxable = is_taxable
        ,is_sale_report = is_sale_report
        ,p_type = p_type
        ,p_tag = p_tag
        ,sku_lst = sku_lst
        ,cost = cost
        ,vendor = vendor
        ,buydown = buydown
        ,breakdown_assoc_lst = breakdown_assoc_lst
    )
    db = couch_db_util.get_store_db(store_id)
    store_product_document.store(db)


# Escreva um programa Python para calcular o número de dias entre dois datetimes. A diferença entre os dias deve ser igual a 10.

from datetime import datetime

dataInicial = str(input('Insira a data inicial: '))
dataFinal = str(input('Insira a data final: '))

dataInicialFormatada = datetime.strptime(dataInicial, '%d/%m/%Y')
dataFinalFormatada = datetime.strptime(dataFinal, '%d/%m/%Y')

quantidadeDias = abs((dataFinalFormatada - dataInicialFormatada).days)

if quantidadeDias == 10:
    print('A quantidade de dias é igual 10')
    print(f'A quantidade de dias foi {quantidadeDias}')
else:
    print('A quantidade de dias é diferente de 10')
    print(f'A quantidade de dias foi {quantidadeDias}')


#!/usr/bin/env python
from setuptools import find_packages, setup

# read the contents of your README file
from pathlib import Path
this_directory = Path(__file__).parent
long_description = (this_directory / "README.md").read_text()

requirements = []
with open("requirements.txt") as f:
    for line in f:
        stripped = line.split("#")[0].strip()
        if len(stripped) > 0:
            requirements.append(stripped)

setup(
    name="scivision",
    version="0.2.8",
    description="scivision",
    author="The Alan Turing Institute",
    author_email="scivision@turing.ac.uk",
    url="https://github.com/alan-turing-institute/scivision",
    packages=find_packages(),
    install_requires=requirements,
    python_requires=">=3.7",
    include_package_data=True,
    long_description=long_description,
    long_description_content_type='text/markdown',
)


import redbean
from redbean import rest_method
import redbean.web.routedef

routes = redbean.RouteTableDef()
import logging

logger = logging.getLogger("hello")
routes = redbean.web.routedef.RouteTableDef(prefix="/api")

@routes.get("/")
@rest_method
async def helloword():
    logger.debug("hi! - debug")
    logger.info("hi! - info")
    logger.warning("hi! - warn")
    logger.error("hi! - error")
    logger.fatal("hi! - final")

    return {"message": "hello world"}

async def create_app():

    app = redbean.Application()
    app.add_routes(routes)
    app.log_routes()

    return app

if __name__ == '__main__':
    redbean.cli(create_app)


from django.shortcuts import render
from django.forms import modelformset_factory
from django.contrib.auth.decorators import login_required
from django.contrib import messages
from django.http import HttpResponse, HttpResponseRedirect
from .forms import  PostForm,adressform
from .models import category,Post

@login_required
def addNoteView(request):
    if request.method =="POST":
        form = PostForm(request.POST,request.FILES)
        if form.is_valid():
            temp=form.save(commit=False)
            temp.user = request.user
            temp.save()

            datas = category.objects.all()
            return render(request,'categoryview.html',{'datas':datas})

        else:

            msg = 'Errors: %s' % form.errors.as_text()
            return HttpResponse(msg, status=400)

    else:
        form = PostForm()
        return render(request,'index.html',{'form':form})

@login_required
def categoryview(request):
    datas = category.objects.all()
    return render(request,'categoryview.html',{'datas':datas})

@login_required
def catdetail(request,id):
    print(type(id))
    if id==int(5):
        posts = Post.objects.all()
        return render(request,'catdetail.html',{'posts':posts})
    else:
        category1 = category.objects.get(pk=id)
        posts= Post.objects.filter(category_id = category1.id)
        return render(request,'catdetail.html',{'posts':posts})

@login_required
def adrform(request):
    if request.method=="POST":
        form = adressform(request.POST)
        if form.is_valid():
            tar = form.save(commit=False)
            tar.user = request.user
            tar.save()
    else:
        form = adressform()
        return render(request,'adressform.html',{'form':form})

def userallpost(request,id):
   posts = Post.objects.filter(user = id)
   return render(request,'alluserpost.html',{'posts':posts})


#-----------------------
# @author: Tony Ribeiro
# @created: 2018/03/10
# @updated: 2018/03/10
#
# Boolean Network Time Series Generator
#
#-----------------------

from __future__ import print_function
import sys
import random

import rule
from utils import eprint, generate_all_transitions, generate_transitions

random.seed(a=None)

# 0) Parameters management
#--------------------------

eprint("Starting Boolean Network Transition Generator...")
eprint("> Parameter extraction")

#******************
# parameters      *
#
input_file = "" 	# $1
semantic = ""		# $2
nb_transitions = 0	# $3
#                 *
#******************

param_id = 1

if len(sys.argv) < 2:
	eprint(">> No input file specified, script terminated")
	sys.exit()

input_file = sys.argv[param_id]
eprint(">> Input file specified: ",input_file)
param_id+=1

if len(sys.argv) > param_id:
	semantic = sys.argv[param_id]
	eprint(">> Semantic given: ",semantic)

	if semantic != "synchronous" and semantic != "asynchronous" and semantic != "general":
		eprint(">>> Bad value for semantic parameter, must be one of the following: synchronous, asynchronous, general")

else:
	eprint(">> No semantic specified, script terminated")
	sys.exit()

param_id+=1

if len(sys.argv) > param_id:
	nb_transitions = int(sys.argv[param_id])
	eprint(">> Number of nb_transitions given: ",nb_transitions)
else:
	eprint(">> No specific number of tranitions requested, all transitions will be generated")

eprint ("> Input file extraction...")

# 1) Extract variables
#------------------------

eprint (">> Extracting variables...")

variables = []
variables_values = []
f = open(input_file,"r")

for line in f:
	tokens = line.split()

	if len(tokens) == 0 or tokens[0] != "VAR":
		break

	variable = tokens[1]
	values = []

	for i in range(2,len(tokens)):
		values.append(tokens[i])

	eprint(">>> Extracted variable:",variable,"domain:",values)

	variables.append(variable)
	variables_values.append(values)

# 2) Extract rules
#------------------------

rules = []

for line in f:
	tokens = line.split()

	if len(tokens) == 0:
		continue

	head = tokens[0]

	# Extract variable
	beg = 0
	end = head.index('(')

	head_variable = head[beg:end]
	head_var_id = variables.index(head_variable)

	# Extract value
	beg = end+1
	end = head.index(',')

	head_value = head[beg:end]
	head_val_id = variables_values[head_var_id].index(head_value)

	eprint("Head extracted: ",head_var_id,"=",head_val_id)

	# Extract body
	body = []

	for i in range(2,len(tokens)):
		condition = tokens[i]

		# Extract variable
		beg = 0
		end = condition.index('(')

		variable = condition[beg:end]
		var_id = variables.index(variable)

		# Extract value
		beg = end+1
		end = condition.index(',')

		value = condition[beg:end]
		val_id = variables_values[var_id].index(value)

		# TODO: delay

		#eprint("Condition extracted: ",variable,"=",value)

		body.append([var_id,val_id,1])

	r = rule.Rule(head_var_id,head_val_id,body)
	rules.append(r)

	eprint("Extracted rule:",r.to_string())

# 3) Time series generation
#---------------------------

eprint ("Generating time series...")

# Print variables
for var in range(0,len(variables)):
	print ("VAR "+variables[var],end='')
	for val in variables_values[var]:
		print(" "+val,end='')
	print()
print()

# Generate all transitions
if nb_transitions == 0:
	initial_state = []

	generate_all_transitions(variables, variables_values, rules, 0, initial_state, semantic)
else:
# Generate given number of transitions:
	generate_transitions(variables, variables_values, rules, semantic, nb_transitions)

#############
# OUTPUT:
# - serie
# - serie_noise
#############

print()


import ply.lex as lex

# Lista de tokens reconhecidos pelo lexer
tokens = [
    'NUMERO_REAL',
    'NUMERO_ROMANO',
    'NUMERO_TELEFONE'
]

# Expressões regulares para reconhecimento de números reais
def t_NUMERO_REAL(t):
    r'-?\d+(\.\d+)?((E|e)[+-]?\d+)?'
    t.value = float(t.value)
    return t

# Expressões regulares para reconhecimento de números romanos
def t_NUMERO_ROMANO(t):
    r'(M{0,3})(CM|CD|D?C{0,3})(XC|XL|L?X{0,3})(IX|IV|V?I{0,3})'
    roman_map = {'I':1, 'V':5, 'X':10, 'L':50, 'C':100, 'D':500, 'M':1000}
    total = 0
    prev = 0
    for c in t.value:
        curr = roman_map[c]
        total += curr
        if curr > prev:
            total -= 2 * prev
        prev = curr
    if total > 1999:
        t.value = 0
    else:
        t.value = total
    return t

# Expressões regulares para reconhecimento de números de telefone
def t_NUMERO_TELEFONE(t):
    r'(\+\d{2})?\s?(\(\d{2}\))?\s?(\d{2})?\s?\d{4,5}[.-]\d{4}'
    t.value = t.value.replace(' ', '')
    t.value = t.value.replace('(', '')
    t.value = t.value.replace(')', '')
    t.value = t.value.replace('.', '')
    t.value = t.value.replace('-', '')
    return t

# Regras de ignorar espaços em branco e outros caracteres indesejados
t_ignore = ' \t\r'

# Função que lida com erros (para fins de debug)
def t_error(t):
    print("Erro: caractere inválido '%s'" % t.value[0])
    t.lexer.skip(1)

# Constrói o lexer
lexer = lex.lex()

# Testa o lexer com uma entrada de exemplo
lexer.input("1.0")
while True:
    tok = lexer.token()
    print(tok)


from bs4 import BeautifulSoup
import requests

while True:
    url = input("Enter source url: ")
    raw_data = requests.get(url)
    soup = BeautifulSoup(raw_data.text, "lxml")
    title = soup.find("title")
    print(title.text)


import dlib
import cv2
import numpy as np
import pymongo
import bson
from scipy.spatial.distance import mahalanobis
from facenet_pytorch import InceptionResnetV1
import torch

class Face_Recognition_Library:
    def __init__(self, DEBUG=False):
        self.client = pymongo.MongoClient("mongodb://localhost:27017/")
        self.db = self.client["miranda"]
        self.collection = self.db["face_features"]

        self.document = {
            "_id": bson.ObjectId(),
            "name": "",
            "embeddings": "",
            "voice_file": "",
        }

        self.model = cv2.dnn.readNetFromTorch("nn4.small2.v1.t7")
        # Load the VGG_Face.t7 pretrained model
        # self.model = InceptionResnetV1(pretrained='vggface2').eval()
        self.predictor = dlib.shape_predictor("shape_predictor_68_face_landmarks.dat")
        self.DEBUG = DEBUG
        self.LEFT_EYE_INDICES = [36, 37, 38, 39, 40, 41]
        self.RIGHT_EYE_INDICES = [42, 43, 44, 45, 46, 47]

    def insert_face(self, name, embeddings=None, FILE=None):
        self.document["embeddings"] = embeddings.tolist()
        # self.document["landmarks"] = [(point.x, point.y) for point in landmarks.parts()]
        self.document["name"] = name

        self.document["voice_file"] = FILE
        self.collection.insert_one(self.document)
        print("Inserted face")

    def get_landmarks(self, img, rect):
        gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

        temp = self.predictor(gray, rect)
        return temp

    def get_embeddings(self, normalized_face):
        blob = cv2.dnn.blobFromImage(normalized_face, 1.0 / 255, (96,96), (0, 0, 0), swapRB=True, crop=False)
        self.model.setInput(blob)
        embeddings = self.model.forward()
        embeddings = embeddings[0]
        return embeddings

        # face = cv2.cvtColor(normalized_face, cv2.COLOR_BGR2RGB)
        # face = np.transpose(face, (2,0,1)).astype(np.float32)
        # face = np.expand_dims(face, axis=0)
        # embeddings = self.model(torch.from_numpy(face))
        # return embeddings.detach().numpy()[0]

    def cosine_similarity(self, a, b):
        return np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b))

    def l2_norm_cosine_similarity(self, face_embedding, target_embedding):
        # L2-normalize the face and target embeddings
        face_embedding = face_embedding / np.linalg.norm(face_embedding)
        target_embedding = target_embedding / np.linalg.norm(target_embedding)

        # Calculate the dot product between the normalized embeddings
        dot_product = np.dot(face_embedding, target_embedding)

        # Calculate the L2-normalized cosine similarity
        similarity = 0.5 * (dot_product + 1)

        return similarity

    def l2_normalized_cosine_similarity(self, embedding1, embedding2):
        # Calculate the dot product between the two vectors
        dot_product = np.dot(embedding1, embedding2)

        # Calculate the L2 norm (i.e., Euclidean length) of each vector
        embedding1_norm = np.linalg.norm(embedding1)
        embedding2_norm = np.linalg.norm(embedding2)

        # Calculate the cosine similarity and L2 normalize the result
        similarity = dot_product / (embedding1_norm * embedding2_norm)
        return similarity

    def recognize_face(self, face_embeddings, target_embedding, threshold=0.5):
        best_match = "unknown"
        best_similarity = -1
        voice_rec = None

        faces = list(self.collection.find({}))
        for face in faces:

            face_embeddings = np.array(face["embeddings"])
            similarity = self.l2_normalized_cosine_similarity(face_embeddings, target_embedding)

            if similarity > best_similarity:
                best_similarity = similarity
                best_match = face["name"]
                voice_rec = face["voice_file"]

        if best_similarity < threshold:
            return "unknown", voice_rec
        else:
            return best_match, voice_rec

    def rect_to_tuple(self, rect):
        left = rect.left()
        right = rect.right()
        top = rect.top()
        bottom = rect.bottom()
        return left, top, right, bottom

    def extract_eye(self, shape, eye_indices):
        points = map(lambda i: shape.part(i), eye_indices)
        return list(points)

    def extract_eye_center(self, shape, eye_indices):
        points = self.extract_eye(shape, eye_indices)
        xs = map(lambda p: p.x, points)
        ys = map(lambda p: p.y, points)
        return sum(xs) // 6, sum(ys) // 6

    def extract_left_eye_center(self, shape):
        return self.extract_eye_center(shape, self.LEFT_EYE_INDICES)

    def extract_right_eye_center(self, shape):
        return self.extract_eye_center(shape, self.RIGHT_EYE_INDICES)

    def angle_between_2_points(self, p1, p2):
        x1, y1 = p1
        x2, y2 = p2
        tan = (y2 - y1) / (x2 - x1)
        return np.degrees(np.arctan(tan))

    def get_rotation_matrix(self, p1, p2):
        angle = self.angle_between_2_points(p1, p2)
        x1, y1 = p1
        x2, y2 = p2
        xc = (x1 + x2) // 2
        yc = (y1 + y2) // 2
        M = cv2.getRotationMatrix2D((xc, yc), angle, 1)
        return M

    def crop_image(self, image, det):
        left, top, right, bottom = self.rect_to_tuple(det)
        return image[top:bottom, left:right]

    def get_normalized_face(self, img, rect):
        scale = 2
        height, width = img.shape[:2]
        s_height, s_width = height // scale, width // scale
        img = cv2.resize(img, (s_width, s_height))
        # Turn rectangle into rectangles for dlib
        # Resize rect
        rect = dlib.rectangle(int(rect.left() / scale), int(rect.top() / scale), int(rect.right() / scale), int(rect.bottom() / scale))

        dets = dlib.rectangles()

        dets.append(rect)

        for i, det in enumerate(dets):
            shape = self.predictor(img, det)
            left_eye = self.extract_left_eye_center(shape)
            right_eye = self.extract_right_eye_center(shape)

            M = self.get_rotation_matrix(left_eye, right_eye)
            rotated = cv2.warpAffine(img, M, (s_width, s_height), flags=cv2.INTER_CUBIC)

            cropped = self.crop_image(rotated, det)

            return cropped


import sys
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import cross_validate
from sklearn.ensemble import GradientBoostingRegressor
from RegressionMetrics import RegressorMetrics

def GradientBoosting(data, labels, nof ,file , filename):
    params = {
        'loss' : ['ls', 'lad', 'huber', 'quantile'],
        'n_estimators' : [10,50,100,200,500],
        'criterion' : ['friedman_mse', 'mse', 'mae'],
        'max_features':['auto', 'sqrt', 'log2'],
        # 'max_depth' : [1,2,3,5,10],
    }
    clf = GridSearchCV(GradientBoostingRegressor(), params, n_jobs=-1, cv=nof)
    clf.fit(data, labels)
    best_parameters = clf.best_params_
    ls = best_parameters['loss']
    ne = best_parameters['n_estimators']
    cr = best_parameters['criterion']
    mf = best_parameters['max_features']
    # md = best_parameters['max_depth']

    model = GradientBoostingRegressor(loss = ls , n_estimators=ne , criterion=cr ,max_features=mf)

    # model = GradientBoostingRegressor(loss = ls , n_estimators=ne , criterion=cr ,max_features=mf , max_depth=md)
    r = RegressorMetrics()
    r.set_predictorName("GradientBoostingRegressor")
    r.set_contin(False)
    r.set_DatasetName(filename)
    bestparameters = 'loss= ' +str(ls) + ' n_estimators= '+str(ne) + ' criterion= '+str(cr) + ' max_features= '+str(mf)
    r.set_best(bestparameters)
    r.PrintbestParameters()

    cv_results = cross_validate(model, data, labels, cv=nof, scoring=r.scorer, return_train_score = True)
    r.Print("GradientBoostingRegressor")
    r.Print("test")
    r.PrintResults("MAE" , cv_results['test_MAE'])
    r.PrintResults("RMAE" , cv_results['test_RMAE'])
    r.PrintResults("MSE" , cv_results['test_MSE'])
    r.PrintResults("RMSE" , cv_results['test_RMSE'])
    r.PrintResults("RS" , cv_results['test_RS'])
    r.Print("train")
    r.PrintResults("MAE" , cv_results['train_MAE'])
    r.PrintResults("RMAE" , cv_results['train_RMAE'])
    r.PrintResults("MSE" , cv_results['train_MSE'])
    r.PrintResults("RMSE" , cv_results['train_RMSE'])
    r.PrintResults("RS" , cv_results['train_RS'])

    # file.write(str(cv_results['test_MAE']))
    # file.write(str(cv_results['test_RMAE']))
    # file.write(str(cv_results['test_MSE']))
    # file.write(str(cv_results['test_RMSE']))
    # file.write(str(cv_results['test_RS']))


import numpy as np
from sklearn.utils import check_random_state
from numbers import Number
import csv
import os
_elements=None

def _load_elements():
	"""
	Loads chemical element information.

	Returns:
	    list: list of lists with the following entries [nuclear charge, element name, valence] for each element
	"""

	atoms_fname= os.path.join(os.path.dirname(__file__),'data/atoms.csv')
	import ast

	with open(atoms_fname) as fh:
		lines=list(csv.reader(fh,delimiter='\t'))[1:]
	for line in lines:
		line[0]=int(line[0]) #Nuclear Charge
		# line[-1]=list(map(int,line[-1].split(', ')))
		line[2]=list(ast.literal_eval(line[2])) #Electorns per shell
		line[3]=int(line[3]) #Empirical Radius of the atom
		line[4]=int(line[4]) #Calculated Radius of the atom
		line[5]=int(line[5]) #van der Waals based radius
		line[6]=int(line[6]) #Covalent radius (single bond)
		line[7]=int(line[7]) #Covalent radius (triple bond)
	return lines

def get_valence(Z):
	"""
	Converts array of nuclear charges to array of corresponding valence.

	Args:
	    Z (numpy ndarray): array with nuclear charges

	Returns:
	    numpy ndarray: array of the same size as Z with the valence of the corresponding atom
	"""
	global _elements
	if _elements is None:
		_elements=_load_elements()
	V=np.array([0]+[line[2][-1] for line in _elements])
	return V[Z]

def get_calculated_radius(Z):
	"""
	Converts array of nuclear charges to array of corresponding valence.

	Args:
	    Z (numpy ndarray): array with nuclear charges

	Returns:
	    numpy ndarray: array of the same size as Z with the valence of the corresponding atom
	"""
	global _elements
	if _elements is None:
		_elements=_load_elements()
	V=np.array([0]+[line[4] for line in _elements])
	ret=V[Z]/100. #angstrom conversion
	# assert (ret>0).all()
	return ret

def get_van_der_waals_radius(Z):
	"""
	Converts array of nuclear charges to array of corresponding valence.

	Args:
	    Z (numpy ndarray): array with nuclear charges

	Returns:
	    numpy ndarray: array of the same size as Z with the valence of the corresponding atom
	"""
	global _elements
	if _elements is None:
		_elements=_load_elements()
	V=np.array([0]+[line[5] for line in _elements])
	ret=V[Z]/100. #angstrom conversion
	# assert (ret>0).all()
	return ret

def get_empirical_radius(Z):
	"""
	Converts array of nuclear charges to array of corresponding valence.

	Args:
	    Z (numpy ndarray): array with nuclear charges

	Returns:
	    numpy ndarray: array of the same size as Z with the valence of the corresponding atom
	"""
	global _elements
	if _elements is None:
		_elements=_load_elements()
	V=np.array([0]+[line[3] for line in _elements])
	ret=V[Z]/100. #angstrom conversion
	# assert (ret>0).all()
	return ret

def get_covalent1_radius(Z):
	"""
	Converts array of nuclear charges to array of corresponding valence.

	Args:
	    Z (numpy ndarray): array with nuclear charges

	Returns:
	    numpy ndarray: array of the same size as Z with the valence of the corresponding atom
	"""
	global _elements
	if _elements is None:
		_elements=_load_elements()
	V=np.array([0]+[line[6] for line in _elements])
	ret=V[Z]/100. #angstrom conversion
	# assert (ret>0).all()
	return ret

def augment(positions, translation_intervals=1,
			rotation_intervals=1, return_translations=False,
			return_rotations=False, random_state=0):
	"""Creates new molecules by applying random rotations and translations to the given atom positions.

	Args:
	    positions (numpy ndarray NxMx3): array of M atom positions for N molecules
	    translation_intervals (int|list, optional): the translations will be made by a uniformly drawn sample in the interval defined by the list or [-number,number]
	    rotation_intervals (number|list, optional): the rotations will be made by a uniformly drawn sample (degrees) in the interval defined by the list or [-number,number]
	    return_translations (bool, optional): if True it returns the translation offsets for each molecule
	    return_rotations (bool, optional): if True it returns the rotation matrices used to rotate each molecule
	    random_state (int, optional): Initial state for random number generator

	Returns:
	    numpy ndarray: array of the same shape as positions with new positions
	    numpy ndarray, optional: [N,3] with the applied translations
	    numpy ndarray, optional: [N,3,3] with the applied translations
	"""
	if isinstance(translation_intervals,Number):
		translation_intervals=(-translation_intervals,translation_intervals)
	if len(translation_intervals)==2:
		translation_intervals=[translation_intervals]*3

	if isinstance(rotation_intervals,Number):
		rotation_intervals=(-rotation_intervals,rotation_intervals)
	if len(rotation_intervals)==2:
		rotation_intervals=[rotation_intervals]*3

	rng=check_random_state(random_state)
	translation_intervals=np.array(translation_intervals)
	translations=rng.rand(positions.shape[0],positions.shape[2])
	translations*=np.diff(translation_intervals,axis=1).ravel()
	translations+=np.array(translation_intervals)[:,0]

	rotation_intervals=np.array(rotation_intervals).astype(np.float)
	rotation_intervals*=(2.*np.pi/360)
	rotations=rng.rand(positions.shape[0],positions.shape[2])
	rotations*=np.diff(rotation_intervals,axis=1).ravel()
	rotations+=np.array(rotation_intervals)[:,0]

	rotation_mats2 = np.zeros((positions.shape[2],rotations.shape[0])+(2,2))
	rotation_mats = np.zeros((positions.shape[2],rotations.shape[0])+(3,3))
	rotation_mats[:,:,[0,1,2],[0,1,2]]=1
	rotation_mats2[:,:,[0,1],[0,1]] =np.cos(rotations.T)[...,np.newaxis]
	rotation_mats2[:,:,0,1] =-np.sin(rotations.T)
	rotation_mats2[:,:,1,0] =np.sin(rotations.T)
	# rotation_mats2[:,:,1,1] =np.cos(rotations.T)
	rotation_mats[0,:,:2,:2]=rotation_mats2[0]
	# rotation_mats[0,:,2,2]=1
	rotation_mats[1,:,::2,::2]=rotation_mats2[1]
	rotation_mats[2,:,1:,1:]=rotation_mats2[2]
	rmats=np.einsum('aij,ajk,akl->ail',*rotation_mats)

	rpos=np.einsum('aij,abj->abi',rmats,positions)
	rpos+=translations[:,np.newaxis,:]
	output = (rpos,)
	if return_translations:
		output= output+(translations,)
	if return_rotations:
		output= output+(rmats,)
	if len(output)==1:
		output=output[0]
	return output


from algorithms.signature_of_fifths_algorithm import SignatureOfFifthsUtility
from model.definitions import AlgorithmInfo, Algorithm
from utils.mappings import create_tonation_string
from algorithms.tonal_profiles_algorithm import TonalProfilesUtility

class AlgorithmManager:
    def __init__(self):
        self.signature_utility = SignatureOfFifthsUtility()
        self.tonal_profiles_utility = TonalProfilesUtility()

    def execute_algorithm(self, algorithm_info: AlgorithmInfo, note_dict):
        result_note, result_mode = None, None
        signature = None
        same_axes = []
        mode_angle_equal_zero = False
        if algorithm_info.algorithm_type == Algorithm.SIGNATURE_MODE_AXIS:
            signature, same_axes = self.basic_signature_algorithm(note_dict)
            tonation, mode_angle_equal_zero = self.signature_utility.calculate_tonation_with_mode_axis(signature)
            result_note, result_mode = tonation.note, tonation.mode
        elif algorithm_info.algorithm_type == Algorithm.SIGNATURE_TONAL_PROFILES:
            signature, same_axes = self.basic_signature_algorithm(note_dict)
            tonation_note = self.signature_utility.get_tonation_pointed_by_mdasf(signature)
            result_note, result_mode = self.tonal_profiles_utility.decide_between_parallel(sample_vector=note_dict,
                                                                                tonation_note=tonation_note,
                                                                                profile=algorithm_info.profile)
        elif algorithm_info.algorithm_type == Algorithm.CLASSIC_TONAL_PROFILES:
            result_note, result_mode = self.tonal_profiles_utility.classic_tonal_profiles_algorithm(
                sample_vector=note_dict, profile=algorithm_info.profile)
        return create_tonation_string(tonation_note=result_note, tonation_mode=result_mode), signature, same_axes, mode_angle_equal_zero

    def basic_signature_algorithm(self, notes_dict):
        signature = self.signature_utility.calculate_signature_of_fifths(notes_dict)
        self.signature_utility.calculate_cvsf(signature)
        same_axes = self.signature_utility.calculate_mdasf(signature)
        return signature, same_axes

    def get_tonation_information(self):
        return self.signature_utility.calculate_tonation(self.signature_of_fifths)


# -*- coding: utf-8 -*-

import json
import logging
import time
import warnings
import functools

from urllib.parse import parse_qs, urlparse, urlencode

# VMware vRealize Log Insight Exporter
# Copyright © 2017 VMware, Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed
# under the License is distributed on an “AS IS” BASIS, without warranties or
# conditions of any kind, EITHER EXPRESS OR IMPLIED. See the License for the
# specific language governing permissions and limitations under the License.

logger = logging.getLogger(__name__)

class SeenWarning(UserWarning):
    """Raised when a @once function is called multiple times with the same arguments."""
    def __init__(self, function_name, args, kwargs):
        self.function_name = function_name
        self.args = args
        self.kwargs = kwargs

    def __str__(self):
        return '{s.function_name}({s.args}, {s.kwargs}) called multiple times'.format(s=self)

def once(func):
    """Decorator: Record function arguments, warn when a function is called multiple times identically."""
    func._seen_history = set()

    @functools.wraps(func)
    def wrapper(*args, **kwargs):
        k = str([args, kwargs])
        if k in func._seen_history:
            warnings.warn(SeenWarning(function_name=func.__name__, args=args, kwargs=kwargs))
        else:
            func._seen_history.add(k)
        return func(*args, **kwargs)
    return wrapper

class ExplorerUrlParse(object):
    """
    Extract query components from an /explorer?existingChartQuery=... url. Does not request the URL.
    Produces three elements, all of which are JSON embedeed in the original URL, and all of which are sparse
        - existingChartQuery - all the helperParams
        - timeWindowToggle - the "1 bar =" bin-width UI element, ignored
        - chartOptions - chart type and legend
    """

    def __init__(self, url):
        self.from_explorer_url(url)
        self.lasttoken = int(time.time()) * 1000

    def from_explorer_url(self, url):
        p = urlparse(url)

        self.scheme = p.scheme
        self.netloc = p.netloc

        query_string = p.query
        queries = parse_qs(query_string)

        def _parse_json_or_empty_dict(queries, s):
            try:
                return json.loads(queries[s][0])
            except:
                logger.debug("Cannot retrieve %s, returning {}" % s)
                return {}

        self.chartOptions = _parse_json_or_empty_dict(queries, 'chartOptions')
        self.timeWindowToggle = _parse_json_or_empty_dict(queries, 'timeWindowToggle')
        self.existingChartQuery = _parse_json_or_empty_dict(queries, 'existingChartQuery')
        self.existingChartQuery_raw = queries['existingChartQuery'][0]

    def currentTimeMillis(self):
        return int(round(time.time() * 1000))

    def getChartParamsHelper(self, extraparams={}):
        warnings.warn(DeprecationWarning("getChartParamsHelper shouldn't be used"))
        """Serialize back to a requests-friendly params dict"""
        token = int(time.time()) * 1000

        while token == self.lasttoken:
            token += 1
            logger.warning("getChartParamsHelper Token bump to {0} in {1}".format(token, id(self)))
        self.lasttoken = token

        params = {}
        if extraparams:
            params.update(extraparams)

        for _ in self.existingChartQuery:

            if _ == 'piqlFunctionGroups':
                insertedkey = 'funcGroups'
            else:
                insertedkey = _

            if self.existingChartQuery[_]:
                s = self.existingChartQuery[_]

                if isinstance(s, dict):
                    # params['paramsHelper.%s' % _] = json.dumps(s)
                    logger.info("Dropping parameter %s = %s" % (_, str(s)))

                elif isinstance(s, list):
                    logger.debug("Mapping list-of-dict item %s = %s" % (_, str(s)))

                    if _ in ["fieldConstraints", "extractedFields"]:
                        counter = 0
                        for listitem in s:
                            for fieldConstraintsDictKey in listitem:
                                params['paramsHelper.%s[%d].%s' % (insertedkey, counter, fieldConstraintsDictKey)] = \
                                    listitem[fieldConstraintsDictKey]
                            counter += 1
                    else:
                        counter = 0
                        for listitem in s:
                            for dictkey in listitem:
                                if listitem[dictkey]:
                                    second_counter = 0
                                    for innerlistitem in listitem[dictkey]:
                                        params['paramsHelper.%s[%d].piqlFunctions[%d]' % (insertedkey, counter, second_counter)] = innerlistitem['value']
                                        second_counter += 1
                            counter += 1

                elif _ == "paramsHelper.cancelToken":
                    logger.info("Dropping paramsHelper.cancelToken from model - should be generated at query-time")
                elif _ == "shouldGroupByTime":
                    params['paramsHelper.%s' % insertedkey] = str(s).lower()
                    logger.debug("Lowercase inserted %s = %s" % (insertedkey, str(s)))
                else:
                    params['paramsHelper.%s' % insertedkey] = str(s)
                    logger.debug("Direct inserted %s = %s" % (insertedkey, str(s)))
            else:
                if _ == "query":
                    params['paramsHelper.%s' % insertedkey] = ""
                    logger.debug("Inserted blank %s = ''" % (insertedkey))

        if 'export' in params:
            params['existingMessageQuery'] = self.existingChartQuery_raw

        return params

    def getExportEventsHelper(self, extraparams={}, altstart=None, altend=None, outputformat='JSON'):
        """Serialize back to a requests-friendly params dict"""
        token = int(time.time()) * 1000

        while token == self.lasttoken:
            token += 1
        self.lasttoken = token

        model = self.existingChartQuery.copy()

        if altstart and altend:
            logger.debug("Applied altstart {0} and altend {1}".format(altstart, altend))
            model['startTimeMillis'] = altstart
            model['endTimeMillis'] = altend
            model['dateFilterPreset'] = 'CUSTOM'
        params = {
            'export': 'true',
            'exportHelper.exportFormat': outputformat,
            'exportHelper.exportToken': token,
            'downloadToken': token,
            'resultTo': 20000,
            'existingMessageQuery': json.dumps(model)
        }
        if extraparams:
            params.update(extraparams)
        return params

    def getExportChartHelper(self, extraparams={}, altstart=None, altend=None):
        """Serialize back to a requests-friendly params dict"""
        token = int(time.time()) * 1000

        while token == self.lasttoken:
            token += 1
        self.lasttoken = token

        params = {}

        for _ in self.existingChartQuery:
            if _ == 'extractedFields':
                continue  # count-over-time charts don't need extracted fields.
            if _ == 'piqlFunctionGroups':
                insertedkey = 'funcGroups'
            else:
                insertedkey = _

            if self.existingChartQuery[_]:
                s = self.existingChartQuery[_]

                if isinstance(s, dict):
                    # params['paramsHelper.%s' % _] = json.dumps(s)
                    logger.debug("Skipping parameter %s = %s" % (_, str(s)))

                elif isinstance(s, list):
                    logger.debug("Mapping list-of-dict item %s = %s" % (_, str(s)))

                    if _ in ["fieldConstraints", "extractedFields"]:
                        counter = 0
                        for listitem in s:
                            for fieldConstraintsDictKey in listitem:
                                params['paramsHelper.%s[%d].%s' % (insertedkey, counter, fieldConstraintsDictKey)] = \
                                    listitem[fieldConstraintsDictKey]
                            counter += 1

                    elif _ in ["groupByFields"]:
                        logger.info("Ignoring group-by fields in chart query: %s" % s)

                    elif insertedkey in ['funcGroups']:
                        for listitem in s:
                            if 'functions' in listitem.keys():
                                logger.info("Ignoring group-by functions in chart query: %s" % listitem)
                        params['paramsHelper.funcGroups[0].piqlFunctions[0]'] = 'COUNT'

                    else:
                        counter = 0
                        for listitem in s:
                            for dictkey in listitem:
                                if listitem[dictkey]:
                                    second_counter = 0
                                    for innerlistitem in listitem[dictkey]:
                                        params['paramsHelper.%s[%d].piqlFunctions[%d]' % (insertedkey, counter, second_counter)] = innerlistitem['value']
                                        second_counter += 1
                            counter += 1

                elif _ == "paramsHelper.cancelToken":
                    logger.info("Dropping paramsHelper.cancelToken from model - should be generated at query-time")
                elif _ == "shouldGroupByTime":
                    params['paramsHelper.%s' % insertedkey] = str(s).lower()
                    logger.debug("Lowercase inserted %s = %s" % (insertedkey, str(s)))
                else:
                    params['paramsHelper.%s' % insertedkey] = str(s)
                    logger.debug("Direct inserted %s = %s" % (insertedkey, str(s)))
            else:
                if _ == "query":
                    params['paramsHelper.%s' % insertedkey] = ""
                    logger.debug("Inserted blank %s = ''" % (insertedkey))

        params.update({
            'export': 'true',
            'exportHelper.exportFormat': 'JSON',
            'exportHelper.exportToken': token,
            'paramsHelper.clientNowMillis': token,
            'downloadToken': token,
        })

        if extraparams:
            params.update(extraparams)

        if altstart and altend:
            logger.debug("Applied altstart {0} and altend {1}".format(altstart, altend))
            params['paramsHelper.startTimeMillis'] = altstart
            params['paramsHelper.endTimeMillis'] = altend
            params['paramsHelper.dateFilterPreset'] = 'CUSTOM'

        return params

    @property
    def start(self):
        return int(self.getExportChartHelper()['paramsHelper.startTimeMillis'])

    @property
    def end(self):
        return int(self.getExportChartHelper()['paramsHelper.endTimeMillis'])

    @property
    def messagesurl(self):
        query = urlencode(self.getChartParamsHelper(extraparams={
            'search': 'true',
            'paramsHelper.clientNowMillis': str(self.currentTimeMillis())
        }))
        return '/messages?' + query

    @property
    def chartingurl(self):
        query = urlencode(self.getChartParamsHelper(extraparams={
            'generateMainChart': 'true',
            'paramsHelper.clientNowMillis': str(self.currentTimeMillis())
        }))
        return '/logcharting?' + query

    @once
    def messagesurl_export(self, altstart=None, altend=None, outputformat=None):
        query = urlencode(self.getExportEventsHelper(altstart=altstart, altend=altend, outputformat=outputformat))
        return '/messages?' + query

    @once
    def chartingurl_export(self, altstart=None, altend=None):
        query = urlencode(self.getExportChartHelper(altstart=altstart, altend=altend))
        return '/logcharting?' + query

    @property
    def groupbyfields(self):
        params = self.getExportChartHelper()
        for p in params:
            if p.startswith('paramsHelper.groupByFields'):
                yield p


"""
Atropos segmentation
"""

__all__ = ['atropos']

import os
import glob
import warnings
from tempfile import mktemp

from ..core import ants_image_io as iio2
from .. import utils

def atropos(a, x, i='Kmeans[3]', m='[0.2,1x1]', c='[5,0]',
            priorweight=0.25, **kwargs):
    """
    A finite mixture modeling (FMM) segmentation approach with possibilities
    for specifying prior constraints. These prior constraints include the
    specification of a prior label image, prior probability images (one for
    each class), and/or an MRF prior to enforce spatial smoothing of the
    labels. Similar algorithms include FAST and SPM. atropos can also perform
    multivariate segmentation if you pass a list of images in: e.g. a=(img1,img2).

    ANTsR function: `atropos`

    Arguments
    ---------
    a : ANTsImage or list/tuple of ANTsImage types
        One or more scalar images to segment. If priors are not used,
        the intensities of the first image are used to order the classes
        in the segmentation output, from lowest to highest intensity. Otherwise
        the order of the classes is dictated by the order of the prior images.

    x : ANTsImage
        mask image.

    i : string
        initialization usually KMeans[N] for N classes or a list of N prior
        probability images. See Atropos in ANTs for full set of options.

    m : string
        mrf parameters as a string, usually "[smoothingFactor,radius]" where
        smoothingFactor determines the amount of smoothing and radius determines
        the MRF neighborhood, as an ANTs style neighborhood vector eg "1x1x1"
        for a 3D image. The radius must match the dimensionality of the image,
        eg 1x1 for 2D and The default in ANTs is smoothingFactor=0.3 and
        radius=1. See Atropos for more options.

    c : string
        convergence parameters, "[numberOfIterations,convergenceThreshold]".
        A threshold of 0 runs the full numberOfIterations, otherwise Atropos
        tests convergence by comparing the mean maximum posterior probability
        over the whole region of interest defined by the mask x.

    priorweight : scalar
        usually 0 (priors used for initialization only), 0.25 or 0.5.

    kwargs : keyword arguments
        more parameters, see Atropos help in ANTs

    Returns
    -------
    dictionary with the following key/value pairs:
        `segmentation`: ANTsImage
            actually segmented image
        `probabilityimages` : list of ANTsImage types
            one image for each segmentation class

    Example
    -------
    >>> import ants
    >>> img = ants.image_read(ants.get_ants_data('r16'))
    >>> img = ants.resample_image(img, (64,64), 1, 0)
    >>> mask = ants.get_mask(img)
    >>> ants.atropos( a = img, m = '[0.2,1x1]', c = '[2,0]',  i = 'kmeans[3]', x = mask )
    >>> seg2 = ants.atropos( a = img, m = '[0.2,1x1]', c = '[2,0]', i = seg['probabilityimages'], x = mask, priorweight=0.25 )
    """
    probs = mktemp(prefix='antsr', suffix='prob%02d.nii.gz')
    tdir = probs.replace(os.path.basename(probs),'')
    probsbase = os.path.basename(probs)
    searchpattern = probsbase.replace('%02d', '*')

    ct = 0
    if isinstance(i, (list,tuple)) and (len(i) > 1):
        while ct < len(i):
            probchar = str(ct+1)
            if ct < 9:
                probchar = '0%s' % probchar
            tempfn = probs.replace('%02d', probchar)
            iio2.image_write(i[ct], tempfn)
            ct += 1
        i = 'PriorProbabilityImages[%s,%s,%s]' % (str(len(i)), probs, str(priorweight))

    if isinstance(a, (list,tuple)):
        outimg = a[0].clone('unsigned int')
    else:
        outimg = a.clone('unsigned int')

    mydim = outimg.dimension
    outs = '[%s,%s]' % (utils._ptrstr(outimg.pointer), probs)
    mymask = x.clone('unsigned int')

    if (not isinstance(a, (list,tuple))) or (len(a) == 1):
        myargs = {
            'd': mydim,
            'a': a,
            'm-MULTINAME-0': m,
            'o': outs,
            'c': c,
            'm-MULTINAME-1': m,
            'i': i,
            'x': mymask
        }
        for k, v in kwargs.items():
            myargs[k] = v

    elif isinstance(a, (list, tuple)):
        if len(a) > 6:
            print('more than 6 input images not really supported, using first 6')
            a = a[:6]
        myargs = {
            'd': mydim,
            'm-MULTINAME-0': m,
            'o': outs,
            'c': c,
            'm-MULTINAME-1': m,
            'i': i,
            'x': mymask
        }
        for aa_idx, aa in enumerate(a):
            myargs['a-MULTINAME-%i'%aa_idx] = aa

    processed_args = utils._int_antsProcessArguments(myargs)
    libfn = utils.get_lib_fn('Atropos')
    retval = libfn(processed_args)

    if retval != 0:
        warnings.warn('ERROR: Non-zero exit status!')

    probsout = glob.glob(os.path.join(tdir,'*'+searchpattern))
    probsout.sort()
    probimgs = [iio2.image_read(probsout[0])]
    for idx in range(1, len(probsout)):
        probimgs.append(iio2.image_read(probsout[idx]))

    outimg = outimg.clone('float')
    return {'segmentation': outimg,
            'probabilityimages': probimgs}


from flask import Flask, render_template, send_file
from prometheus_client import start_http_server, Gauge
import time, psutil, threading

cpu_usage_gauge = Gauge('cpu_usage', 'CPU_Usage')
ram_usage_gauge = Gauge('ram_usage', 'Ram_Usage')
running_time_gauge = Gauge('running_time', 'Running Time')
start_time = time.time()

app = Flask(__name__)

def update_metrics():
    while True:
        elapsed_time = time.time() - start_time
        cpu_usage_gauge.set(psutil.cpu_percent())
        ram_usage_gauge.set(psutil.virtual_memory().percent)
        running_time_gauge.set(int(elapsed_time))
        print("Data Output:\n CPU OUTPUT: "+ str(cpu_usage_gauge)+" RAM OUTPUT: "+str(ram_usage_gauge))
        time.sleep(1)

@app.route('/')
def home():
    return render_template('home.html')

@app.route('/about')
def about():
    return render_template('about.html')

@app.route('/graphics')
def graphics():
    return render_template('graphics.html')

@app.route('/streaming')
def streaming():
    video_path = 'static/video/video_1.mp4'
    return send_file(video_path, mimetype='video/mp4')

if __name__ == '__main__':

    metric_update_thread = threading.Thread(target=update_metrics)
    metric_update_thread.daemon = True
    metric_update_thread.start()

    start_http_server(8000)

    app.run(host='0.0.0.0', port=8082)