from django import forms
from libstat.models import Variable
from libstat.utils import VARIABLE_TYPES, SURVEY_TARGET_GROUPS

__author__ = "vlovgr"

class VariableForm(forms.Form):
    active_from = forms.DateField(
        required=False, widget=forms.TextInput(attrs={"class": "form-control"})
    )
    active_to = forms.DateField(
        required=False, widget=forms.TextInput(attrs={"class": "form-control"})
    )

    question = forms.CharField(
        required=False, widget=forms.TextInput(attrs={"class": "form-control"})
    )
    question_part = forms.CharField(
        required=False, widget=forms.TextInput(attrs={"class": "form-control"})
    )
    category = forms.CharField(
        required=False, widget=forms.TextInput(attrs={"class": "form-control"})
    )
    sub_category = forms.CharField(
        required=False, widget=forms.TextInput(attrs={"class": "form-control"})
    )

    type = forms.ChoiceField(required=True, widget=forms.RadioSelect())

    # Since this is a checkbox, a value will only be returned in form if the
    # checkbox is checked. Hence the required=False.
    is_public = forms.BooleanField(
        required=False, widget=forms.CheckboxInput(attrs={"value": "1"})
    )

    target_groups = forms.MultipleChoiceField(
        required=True, widget=forms.CheckboxSelectMultiple()
    )

    description = forms.CharField(
        widget=forms.Textarea(attrs={"class": "form-control", "rows": "2"})
    )
    comment = forms.CharField(
        required=False,
        widget=forms.Textarea(attrs={"class": "form-control", "rows": "2"}),
    )

    replaces = forms.CharField(
        required=False, widget=forms.TextInput(attrs={"class": "form-control"})
    )

    def __init__(self, *args, **kwargs):
        self.instance = kwargs.pop("instance", None)
        super(VariableForm, self).__init__(*args, **kwargs)
        if not self.instance:
            self.fields["key"] = forms.CharField(
                required=True, widget=forms.TextInput(attrs={"class": "form-control"})
            )

        self.fields["type"].choices = [type for type in VARIABLE_TYPES]
        self.fields["target_groups"].choices = [
            target_group for target_group in SURVEY_TARGET_GROUPS
        ]

        if self.instance:
            self.fields["active_from"].initial = (
                self.instance.active_from.date() if self.instance.active_from else None
            )
            self.fields["active_to"].initial = (
                self.instance.active_to.date() if self.instance.active_to else None
            )
            self.fields["question"].initial = self.instance.question
            self.fields["question_part"].initial = self.instance.question_part
            self.fields["category"].initial = self.instance.category
            self.fields["sub_category"].initial = self.instance.sub_category
            self.fields["type"].initial = self.instance.type
            self.fields["is_public"].initial = self.instance.is_public
            self.fields["target_groups"].initial = self.instance.target_groups
            self.fields["description"].initial = self.instance.description
            self.fields["comment"].initial = self.instance.comment
            self.fields["replaces"].initial = (
                ", ".join([str(v.id) for v in self.instance.replaces])
                if self.instance.replaces
                else ""
            )
            self.replaces_initial_value = ", ".join(
                ["{}:{}".format(v.key, str(v.id)) for v in self.instance.replaces]
                if self.instance.replaces
                else []
            )

    def clean(self):
        cleaned_data = super(VariableForm, self).clean()

        replaces = cleaned_data["replaces"] if "replaces" in cleaned_data else None
        active_from = (
            cleaned_data["active_from"] if "active_from" in cleaned_data else None
        )
        if replaces and not active_from:
            self._errors["replaces"] = self.error_class(
                ["Ange när ersättning börjar gälla genom att sätta 'Giltig fr o m'"]
            )
            self._errors["active_from"] = self.error_class(["Måste anges"])

            del cleaned_data["replaces"]
            del cleaned_data["active_from"]

        active_to = cleaned_data["active_to"] if "active_to" in cleaned_data else None
        if (
            self.instance
            and self.instance.replaced_by
            and active_to
            and self.instance.active_to
            and active_to != self.instance.active_to.date()
        ):
            self._errors["active_to"] = self.error_class(["Styrs av ersättande term"])
            del cleaned_data["active_to"]
        return cleaned_data

    def save(self, commit=True, user=None, activate=False):
        variable = self.instance if self.instance else Variable(is_draft=True)
        variable.key = self.instance.key if self.instance else self.cleaned_data["key"]
        variable.active_from = self.cleaned_data[
            "active_from"
        ]  # Need to convert to UTC? It's a date and not a datetime...
        variable.active_to = (
            self.instance.active_to
            if self.instance and self.instance.replaced_by
            else self.cleaned_data["active_to"]
        )
        variable.question = self.cleaned_data["question"]
        variable.question_part = self.cleaned_data["question_part"]
        variable.category = self.cleaned_data["category"]
        variable.sub_category = self.cleaned_data["sub_category"]
        variable.type = self.cleaned_data["type"]
        variable.is_public = self.cleaned_data["is_public"]
        variable.target_groups = self.cleaned_data["target_groups"]
        variable.description = self.cleaned_data["description"]
        variable.comment = self.cleaned_data["comment"]
        if activate:
            variable.is_draft = False

        to_replace = (
            self.cleaned_data["replaces"].split(", ")
            if self.cleaned_data["replaces"]
            else []
        )
        modified_siblings = variable.replace_siblings(
            to_replace, switchover_date=variable.active_from
        )

        if commit:
            variable.save_updated_self_and_modified_replaced(modified_siblings)

        return variable


import simpy

from settings import *
from simulation import *
from sprites import *
from window import Window

from in_progress import Airline

# create a simulation environment
environ = simpy.Environment(initial_time=0)

# create an arrival area in this environment
arrival_area = arrival(environ)

# create the window and link both ways for convenience
# a priori, only one to one GUI to sim so it's alright
win = Window(arrival_area, CONFIG)
arrival_area.link_GUI(win)

counter_table = (
    [4 for i in range(5)]
    + [0 for i in range(5)]
    + [4 for i in range(10)]
    + [20 for i in range(2000)]
)
counter_table2 = (
    [2 for i in range(10)]
    + [4 for i in range(10)]
    + [60 for i in range(20)]
    + [20 for i in range(2000)]
)

airline_codes = ["AF", "VN"]

airline = Airline(
    win,
    airline_codes[0],
    vec(0, 0),
    counter_table,
)

airline = Airline(
    win,
    airline_codes[1],
    vec(0, 15),
    counter_table2,
)

process_list = [
    "{}_checkin_desk".format(airline_codes[0]),
    "{}_checkin_desk".format(airline_codes[1]),
]
arrival_area.env.process(Pax_generator(arrival_area, 5, 600 / 60, process_list))

# run!
win.run(steps_per_update=STEPS_PER_UPDATE)


from xml.etree.ElementTree import Element, SubElement, ElementTree

#首先创建根节点
root = Element('lab')

#添加子节点SubElement(父节点Element对象， Tag字符串格式， Attribute字典格式)
person1 = SubElement(root, 'person', {'name':'Blue'})

#添加子节点
age1 = SubElement(person1, 'age')
#添加text，即22，字符串格式
age1.text = '22'
gender1 = SubElement(person1, 'gender')
gender1.text = 'male'

person2 = SubElement(root, 'person', {'name':'Yellow'})
age2 = SubElement(person2, 'age')
age2.text = '20'
gender2 = SubElement(person2, 'gender')
gender2.text = 'female'

#将根目录转化为xml树状结构(即ElementTree对象)
tree = ElementTree(root)
#写入xml文件
tree.write('sample.xml', encoding="utf-8", xml_declaration=True)


# Extract Unsubscribe Links
# By Vanesa Hercules, 2022

import gspread
from bs4 import BeautifulSoup
import re
import pandas as pd

# Assign Service Account Key
gc = gspread.service_account(filename='client_secret.json')

# Open Google Sheet
sh = gc.open("Unsubscribe Gmail Messages")

# Get All Values From Body Column
values_list = sh.sheet1.col_values(4)

# Create List of Output Cells Per Length of value_ list
cell_list = []
for cell in range(1, len(values_list)+1):
    cell_str = str(cell)
    cell_list.append('E'+ cell_str)

# Zip cell_list with values_list into a DataFrame
df = pd.DataFrame(list(zip(cell_list, values_list)), columns =['cell', 'body'])

# Search for unsubscribe links and write them back to Google Sheet
def search_for_unsublink (body):
    """ Searches for 'unsubscribe' text among all a tags and returns href if found or none"""
    soup = BeautifulSoup(body, features='lxml')
    a_list = list(soup.find_all('a'))
    for a in a_list:
        if re.search('nsubscribe', str(a)):
            unsubscribe_a = a
            unsubscribe_link = unsubscribe_a.get('href')
            return unsubscribe_link

def write_link_to_cell(cell, link):
    """ Write links to Google Sheet"""
    sh.sheet1.update(cell, link)

# Print iterations progress
def printProgressBar (iteration, total, prefix = '', suffix = '', decimals = 1, length = 100, fill = '█', printEnd = "\r"):
    """
    Call in a loop to create terminal progress bar
    @params:
        iteration   - Required  : current iteration (Int)
        total       - Required  : total iterations (Int)
        prefix      - Optional  : prefix string (Str)
        suffix      - Optional  : suffix string (Str)
        decimals    - Optional  : positive number of decimals in percent complete (Int)
        length      - Optional  : character length of bar (Int)
        fill        - Optional  : bar fill character (Str)
        printEnd    - Optional  : end character (e.g. "\r", "\r\n") (Str)
    """
    percent = ("{0:." + str(decimals) + "f}").format(100 * (iteration / float(total)))
    filledLength = int(length * iteration // total)
    bar = fill * filledLength + '-' * (length - filledLength)
    print(f'\r{prefix} |{bar}| {percent}% {suffix}', end = printEnd)
    # Print New Line on Complete
    if iteration == total:
        print()

# Main
l = len(df)
for index, row in df.iterrows():
    result = search_for_unsublink(row['body'])
    write_link_to_cell(row['cell'], result)
    # Update Progress Bar
    printProgressBar(index + 1, l, prefix = 'Progress:', suffix = 'Complete', length = 50)


import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
from sklearn.metrics import confusion_matrix
from tensorflow.examples.tutorials.mnist import input_data

'''initial steps'''
#tensorflow loading mnist data
#loading data as one hot vector
'''one hot vector-it means data labels (classes) is converted into columns representing binary representation of labels.same number
of rows but but number of columns is equal to number of classes.(0 or 1 representation)'''

data=input_data.read_data_sets("data/MNIST/", one_hot=True)

print("size od data loaded:")
print("Training set size:\t {}".format(len(data.train.labels)))
print("Test set size:\t {}".format(len(data.test.labels)))
print("Validation set size:\t {}".format(len(data.validation.labels)))
#index in one hot vector represents actual class whether its number 1,2,...8,9.
data.test.cls=np.array([label.argmax() for label in data.test.labels])

#default mnist image size is 28 by 28:
img_size=28

#storing images in one dimensional array like a onr dimensional vector represntation.length of one dim array[28*28]
img_flat=img_size*img_size

#shape of image with heigth,width
img_shape=(img_size,img_size)

#number of classses
no_of_classes=10

def plot_images(images, cls_true, cls_pred=None):
    assert len(images) == len(cls_true) == 9

    # Create figure with 3x3 sub-plots.
    fig, axes = plt.subplots(3, 3)
    fig.subplots_adjust(hspace=0.3, wspace=0.3)

    for i, ax in enumerate(axes.flat):
        # Plot image.
        ax.imshow(images[i].reshape(img_shape), cmap='binary')

        # Show true and predicted classes.
        if cls_pred is None:
            xlabel = "True: {0}".format(cls_true[i])
        else:
            xlabel = "True: {0}, Pred: {1}".format(cls_true[i], cls_pred[i])

        ax.set_xlabel(xlabel)

        # Remove ticks from the plot.
        ax.set_xticks([])
        ax.set_yticks([])
    plt.show()

# Get the first images from the test-set.
images = data.test.images[0:9]

# Get the true classes for those images.
cls_true = data.test.cls[0:9]

# Plot the images and labels using our helper-function above.
plot_images(images=images, cls_true=cls_true)

#none means it will accept any number of input images with each vector representing size of 28 by 28.
x=tf.placeholder(tf.float32,[None,img_flat])

y_true=tf.placeholder(tf.int64,[None,no_of_classes])

y_true_cls = tf.placeholder(tf.int64, [None])

#2-D tensor representing weights
weights=tf.Variable(tf.zeros([img_flat,no_of_classes]))

#1-D tensor representing biases
biases=tf.Variable(tf.zeros([no_of_classes]))

#now trying to build a  model
#the element of the ith row and jth column is an estimate of how likely the ith input image is to be of the jth class.
logits = tf.matmul(x, weights) + biases

y_pred = tf.nn.softmax(logits)

#predicting the label
y_pred_cls = tf.argmax(y_pred, dimension=1)

#cost function calculation.i.e. comparing true labels with predicted labels.
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=logits,labels=y_true)

#in earlier step cross_entropy was calculated for each image.Now use all cross_entropy values and calculate its means to know
#how our model well our model is evaluating.
cost = tf.reduce_mean(cross_entropy)

optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.5).minimize(cost)

correct_prediction = tf.equal(y_pred_cls, y_true_cls)
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

#########################################################################################

'''Here tensorflow graph generation code ends '''

'''Actual working starts from below'''
#########################################################################################
session = tf.Session()
#this below line intializes the tensorflow weights and biases.You need to initialize them otherwise error would be caused.
session.run(tf.global_variables_initializer())
batch_size=1000

#actual working of code of above graph created starts from here
def optimize(number_of_iterations):
	for i in range(number_of_iterations):
		x_batch, y_true_batch = data.train.next_batch(batch_size)
		feed_dict_train = {x: x_batch,y_true: y_true_batch}
		session.run(optimizer, feed_dict=feed_dict_train)

feed_dict_test = {x: data.test.images,y_true: data.test.labels,y_true_cls: data.test.cls}

def print_accuracy():
    # Use TensorFlow to compute the accuracy.
	acc = session.run(accuracy, feed_dict=feed_dict_test)

    # Print the accuracy.
	print("Accuracy on test-set: {0:.1%}".format(acc))

def plot_example_errors():
	# Use TensorFlow to get a list of boolean values
	# whether each test-image has been correctly classified,
	# and a list for the predicted class of each image.
	correct, cls_pred = session.run([correct_prediction, y_pred_cls],
									feed_dict=feed_dict_test)

	# Negate the boolean array.
	incorrect = (correct == False)

	# Get the images from the test-set that have been
	# incorrectly classified.
	images = data.test.images[incorrect]

	# Get the predicted classes for those images.
	cls_pred = cls_pred[incorrect]

	# Get the true classes for those images.
	cls_true = data.test.cls[incorrect]

	# Plot the first 9 images.
	plot_images(images=images[0:9],cls_true=cls_true[0:9],cls_pred=cls_pred[0:9])

def plot_weights():
	# Get the values for the weights from the TensorFlow variable.
	w = session.run(weights)

	# Get the lowest and highest values for the weights.
	# This is used to correct the colour intensity across
	# the images so they can be compared with each other.
	w_min = np.min(w)
	w_max = np.max(w)

	# Create figure with 3x4 sub-plots,
	# where the last 2 sub-plots are unused.
	fig, axes = plt.subplots(3, 4)
	fig.subplots_adjust(hspace=0.3, wspace=0.3)

	for i, ax in enumerate(axes.flat):
		# Only use the weights for the first 10 sub-plots.
		if i<10:
			# Get the weights for the i'th digit and reshape it.
			# Note that w.shape == (img_size_flat, 10)
			image = w[:, i].reshape(img_shape)

			# Set the label for the sub-plot.
			ax.set_xlabel("Weights: {0}".format(i))

			# Plot the image.
			ax.imshow(image, vmin=w_min, vmax=w_max, cmap='seismic')

		# Remove ticks from each sub-plot.
		ax.set_xticks([])
		ax.set_yticks([])
	plt.show()

optimize(30)
print_accuracy()

plot_example_errors()


import numpy
import matplotlib.patches as ptch
from PlottingExtensions import Style as sty

def get_aspect(ax):
    figW, figH = ax.get_figure().get_size_inches()
    _, _, w, h = ax.get_position().bounds
    disp_ratio = (figH * h) / (figW * w)
    xlim = ax.get_xlim()
    ylim = ax.get_ylim()
    data_ratio = (ylim[1]-ylim[0]) / (xlim[1]-xlim[0])
    return disp_ratio / data_ratio

##
def polygon(ax, x, y, facecolor=sty.color.lightgrey, edgecolor=sty.color.darkgrey, **kwargs):
    poly = ptch.Polygon([*zip(x,y)], facecolor=facecolor, edgecolor=edgecolor, **kwargs)
    ax.add_patch(poly)

def arrow_head(ax, xy_tip, height, width, direction='up', angle=None, color=sty.color.black, gid='', **kwargs):
    def A(phi, r):
        return numpy.matrix([[numpy.cos(phi)   , -numpy.sin(phi)*r],
                             [numpy.sin(phi)/r ,  numpy.cos(phi)  ]])

    def arrow_polygon(xy_tip, heigth, width, angle, xy_aspect):
        p_tip = numpy.matrix([[xy_tip[0]], [xy_tip[1]]])
        p = numpy.matrix([numpy.array([-height, 0, -height]),
                          numpy.array([width/2, 0, -width/2])])
        return A(angle, xy_aspect)*p + p_tip

    def get_patch(polygon, color, gid, **kwargs):
        x = numpy.array(polygon[0][:]).flatten()
        y = numpy.array(polygon[1][:]).flatten()
        if 'linewidth' in kwargs.keys(): kwargs.pop('linewidth')
        return ptch.Polygon([*zip(x, y)], color=color, linewidth=0, gid=gid, **kwargs)

    def get_angle(direction):
        if direction == 'up':
            return numpy.pi/2
        elif direction == 'left':
            return numpy.pi
        elif direction == 'down':
            return -numpy.pi/2
        else:
            return 0

    if angle == None: angle = get_angle(direction)
    polygon = arrow_polygon(xy_tip, height, width, angle, get_aspect(ax))
    patch = get_patch(polygon, color, gid, **kwargs)
    ax.add_patch(patch)

def arrow(ax, xy0, xy1, rel_height=0.15, rel_width=0.13, color=sty.color.black, gid='', **kwargs):
    dx = xy1[0]-xy0[0]
    dy = xy1[1]-xy0[1]
    length = numpy.sqrt(dx**2 + dy**2)
    angle = numpy.arctan2(dy, dx)
    arrow_head(
        ax=ax,
        xy_tip=xy1,
        height=rel_height*length,
        width=rel_width*length,
        angle=angle,
        color=color,
        gid=gid+'_head',
        **kwargs)

    base_length = (1-rel_height)*length
    base_dx = base_length*numpy.cos(angle)
    base_dy = base_length*numpy.sin(angle)
    ax.plot([xy0[0], xy0[0]+base_dx], [xy0[1], xy0[1]+base_dy], color=color, gid=gid+'_base', **kwargs)

def cursor(ax, xy, r=1, color=sty.color.black, gid=''):
    patch = ptch.Ellipse(xy, r, r/get_aspect(ax), color=color, gid=gid, clip_on=False, zorder=100)
    ax.add_patch(patch)

def coordinate_system(ax, x_arrow_length=0.05, x_arrow_base=0.1, neg_arrow=True, gid_ext=''):
    def move_spines(ax, xlim, ylim, arrow_length):
        ax.spines['right'].set_visible(False)
        ax.spines['top'].set_visible(False)
        ax.spines['bottom'].set_position('zero')
        ax.spines['left'].set_position('zero')
        ax.spines['bottom'].set_gid('x_axis{:s}'.format(gid_ext))
        ax.spines['left'].set_gid('y_axis{:s}'.format(gid_ext))
        if xlim[0] < 0:
            ax.spines['bottom'].set_bounds(xlim[0]+arrow_length, xlim[1]-arrow_length)
        else:
            ax.spines['bottom'].set_bounds(0, xlim[1]-arrow_length)
        if ylim[0] < 0:
            ax.spines['left'].set_bounds(ylim[0]+arrow_length/get_aspect(ax), ylim[1]-arrow_length/get_aspect(ax))
        else:
            ax.spines['left'].set_bounds(0, ylim[1]-arrow_length/get_aspect(ax))

    xlim = ax.get_xlim()
    ylim = ax.get_ylim()
    if xlim[0] > 0:
        xlim[0] = 0
        ax.set_xlim(xlim)
    if ylim[0] > 0:
        ylim[0] = 0
        ax.set_ylim(ylim)
    width = xlim[1]-xlim[0]

    arrow_height = width*x_arrow_length
    arrow_width = width*x_arrow_base

    move_spines(ax, xlim, ylim, arrow_height)
    arrow_head(ax, (xlim[1], 0), arrow_height, arrow_width, direction='right', clip_on=False, gid='x_axis_pos_head{:s}'.format(gid_ext))
    arrow_head(ax, (0, ylim[1]), arrow_height, arrow_width, direction='up', clip_on=False, gid='y_axis_pos_head{:s}'.format(gid_ext))
    if xlim[0] < 0 and neg_arrow:
        arrow_head(ax, (xlim[0], 0), arrow_height, arrow_width, direction='left', clip_on=False, gid='x_axis_neg_head{:s}'.format(gid_ext))
    if ylim[0] < 0 and neg_arrow:
        arrow_head(ax, (0, ylim[0]), arrow_height, arrow_width, direction='down', clip_on=False, gid='y_axis_neg_head{:s}'.format(gid_ext))

def horizontal_legend(ax, **kwargs):
    ax.legend(
        bbox_to_anchor=(0., 1.02, 1., .102),
        loc='lower center',
        ncol=len(ax.get_legend_handles_labels()[0]),
        borderaxespad=0.,
        frameon=False,
        **kwargs
        )


import logging
import traceback

from flask_restplus import Api
import settings

log = logging.getLogger(__name__)

api = Api(version='1.0.1',
          title='Stockkly API',
          description='A free, near real-time, RestAPI for providing price data for stocks (FTSE, NASDAQ), crypto-currencies, funds and Precious metals (GOLD, SILVER).')

@api.errorhandler
def default_error_handler(e):
    message = 'An unhandled exception occurred.'
    log.exception(message)

    if not settings.FLASK_DEBUG:
        return {'message': message}, 500


import logging
import os
import random
import re
from collections import defaultdict
from statistics import mean

import torch
import torchvision.transforms.functional as TF
from torch.utils.data import RandomSampler, SequentialSampler
from torchvision.utils import draw_bounding_boxes

from dataset.utils import WordVocab, ProgramVocab
from utils import join, mkdir, underscores, camel_case, num2word
from utils import to_cpu_detach, nonzero

class Dataset(torch.utils.data.Dataset):
    DATASET_REGISTRY = {}
    image_size = (224, 224)
    split2spec = {"train": 0, "val": 1, "test": 2}
    period_map = {"train": 1, "val": 1, "test": 1}
    concept_maps_ = {}
    max_question_length = 50

    @property
    def is_builder(self):
        return 'Builder' in self.__class__.__name__

    @property
    def is_support(self):
        return 'Support' in self.__class__.__name__

    @property
    def is_debias(self):
        return 'Debias' in self.__class__.__name__

    @staticmethod
    def get_name(name):
        return underscores(name[:-len("Dataset")])

    def get_augmented_name(self, name):
        name = underscores(name[:-len("Dataset")])
        if self.shot_k > 1: name = name.replace('few', num2word(self.shot_k))
        return name

    @property
    def base_dataset(self):
        return underscores(self.__class__.__name__).split('_')[0]

    def __init_subclass__(cls, **kwargs):
        super().__init_subclass__(**kwargs)
        name = Dataset.get_name(cls.__name__)
        cls.DATASET_REGISTRY[name] = cls
        cls.name = name

    def __str__(self):
        return f"{self.name}_{self.split}"

    @property
    def kinds(self):
        return torch.arange(len(self.kinds_))

    @property
    def tag(self):
        if self.split in ["train", "val"]:
            return self.split
        else:
            return f"{self}"

    @property
    def sampler(self):
        if self.split in ["train", "val"]:
            sampler = RandomSampler(self)
        else:
            sampler = SequentialSampler(self)
        return sampler

    def get_batch_sampler(self, batch_size):
        return None

    def select_split(self, split_spec):
        if self.split == "all":
            return list(range(len(split_spec)))
        else:
            return nonzero(split_spec == self.split2spec[self.split])

    @property
    def principal_metric(self):
        return "accuracy"

    def __init__(self, cfg, args):
        self.root = cfg.ROOT
        self.has_mask = cfg.HAS_MASK
        self.shot_k = cfg.SHOT_K
        self.query_k = cfg.QUERY_K
        self.dropout_rate = cfg.DROPOUT_RATE
        self.use_text = cfg.USE_TEXT
        self.opts = cfg.OPTS

        self.args = args
        self.split = cfg.SPLIT
        self.split_ratio = cfg.SPLIT_RATIO
        self.indices_split = []
        self.split_specs = torch.LongTensor([])
        self.kinds_ = []
        self.concepts_ = []
        self.concepts = []
        self.concept2kinds = torch.LongTensor([])
        self.named_entries_ = []
        self.names = []
        self.synonyms = {}
        self.entry2kinds = torch.LongTensor([])
        self.entry2idx_ = {}

        self.image_split_specs = torch.LongTensor([])
        self.concept_split_specs = torch.LongTensor([])
        self.itself2samekinds = {}

        self.transform = self.transform_fn(self.image_size)
        self.word_vocab = None
        self.program_vocab = None
        self.split_seed = cfg.SPLIT_SEED
        self.iteration = 0
        self.mode = "default"

    @property
    def augmented_root(self):
        root_head, root_tail = os.path.split(self.root)
        assert os.access(root_head, os.W_OK)
        return mkdir(join(root_head, '.augmented', root_tail, str(self.split_seed)))

    @property
    def is_alt(self):
        return 'alt' in self.opts

    @property
    def is_shallow(self):
        return 'shallow' in self.opts

    def log_info(self):
        logger = logging.getLogger("falcon_logger")
        logger.warning(f"{self} dataset uses {len(self.indices_split)} out of "
                       f"{len(self.split_specs)} samples.")
        logger.info(f"{self} has {len(self.kinds)} kinds.")
        logger.info(f"{self} has a word vocabulary size of {len(self.word_vocab)}.")
        if self.program_vocab is not None:
            logger.info(f"{self} has a program vocabulary size of {len(self.program_vocab)}.")

    @property
    def info(self):
        return {"concept_entries": len(self.named_entries_), "kind_entries": len(self.kinds_),
            "split": self.split, 'use_text': self.use_text}

    @property
    def empty_image(self):
        return torch.zeros((6 if self.has_mask else 3, *self.image_size))

    @property
    def transform_fn(self):
        raise NotImplementedError

    def get_stacked_scenes(self, image_index):
        raise NotImplementedError

    def get_image(self, image_index):
        raise NotImplementedError

    def get_mask(self, image_index):
        raise NotImplementedError

    def get_annotated_image(self, image_index, mask_index=None):
        masks, image = self.transform(self.get_mask(image_index), self.get_image(image_index))
        if mask_index is not None:
            masks = masks[mask_index:mask_index + 1]
        y_sum = masks.sum(-2).bool()
        y_min_value, y_min = y_sum.max(-1)
        y_max = y_sum.shape[-1] - 1 - y_sum.flip(-1).max(-1).indices
        x_sum = masks.sum(-1).bool()
        x_min_value, x_min = x_sum.max(-1)
        x_max = x_sum.shape[-1] - 1 - x_sum.flip(-1).max(-1).indices
        bboxes = torch.stack([y_min, x_min, y_max, x_max], -1)
        bboxes = bboxes[y_min_value & x_min_value]
        annotated_image = draw_bounding_boxes((image * 255).to(torch.uint8), bboxes, width=1) / 255
        return TF.resize(annotated_image, self.image_size)

    def __len__(self):
        return len(self.indices_split)

    def init_evaluate(self, mode):
        if mode is None: mode = self.mode
        return defaultdict(list, mode=mode)

    def batch_evaluate(self, inputs, outputs, evaluated):
        inputs = to_cpu_detach(inputs)
        outputs = to_cpu_detach(outputs)
        handler = getattr(self, f"batch_{evaluated['mode']}_handler", self.batch_default_handler)
        for k, v in handler(inputs, outputs).items():
            evaluated[k].extend(v)

    def evaluate_metric(self, evaluated):
        handler = getattr(self, f"metric_{evaluated['mode']}_handler", self.metric_default_handler)
        metrics = handler(evaluated)
        if self.principal_metric in metrics: metrics['principal'] = metrics[self.principal_metric]
        return  metrics

    def save(self, output_dir, evaluated, checkpointer, metrics):
        output_dir = mkdir(join(output_dir, "inference", str(self)))
        handler = getattr(self, f"save_{evaluated['mode']}_handler", self.save_default_handler)
        handler(output_dir, evaluated, checkpointer, metrics)

    def batch_default_handler(self, inputs, outputs):
        return {}

    def metric_default_handler(self, evaluated):
        return {}

    def save_default_handler(self, output_dir, evaluated, iteration, metrics):
        pass

    def callback(self, iteration):
        pass

    def _build_word_tokens(self):
        raise NotImplementedError

    def _build_word_vocab(self):
        word_tokens = self._build_word_tokens()
        vocab = WordVocab()
        vocab.words.update(word_tokens)
        vocab.freeze()
        return vocab

    def _build_program_tokens(self):
        raise NotImplementedError

    def _build_program_vocab(self):
        program_tokens = self._build_program_tokens()
        vocab = ProgramVocab()
        vocab.words.update(program_tokens)
        vocab.freeze()
        return vocab

    @staticmethod
    def answer2text(target):
        assert target is not None
        if isinstance(target, bool):
            return "Yes" if target else "No"
        else:
            return str(target)

    def answer2category(self, target):
        if self.use_text:
            return "token"
        elif isinstance(target, str):
            return "choice"
        elif isinstance(target, bool):
            return "boolean"
        else:
            return "count"

    @staticmethod
    def answer2target(answer):
        return torch.tensor(answer).float()

    @staticmethod
    def clevr2nscl(clevr_program):
        nscl_program = list()
        mapping = dict()

        for block_id, block in enumerate(clevr_program):
            op = block['type'] if 'type' in block else block['function']
            current = None
            if op == 'scene':
                current = dict(op='scene')
            elif op.startswith('filter'):
                concept = block['value_inputs'][0]
                last = nscl_program[mapping[block['inputs'][0]]]
                if last['op'] == 'filter':
                    last['concept'].append(concept)
                else:
                    current = dict(op='filter', concept=[concept])
            elif op.startswith('relate'):
                concept = block['value_inputs'][0]
                current = dict(op='relate', relational_concept=[concept])
            elif op.startswith('same'):
                current = dict(op='relate_attribute_equal', attribute=(op.split('_')[1]))
            elif op in ('intersect', 'union'):
                current = dict(op=op)
            elif op == 'unique':
                pass  # We will ignore the unique operations.
            else:
                if op.startswith('query'):
                    if block_id == len(clevr_program) - 1:
                        current = dict(op='query', attribute=(op.split('_')[1]))
                elif op.startswith('equal') and op != 'equal_integer':
                    current = dict(op='query_attribute_equal', attribute=(op.split('_')[1]))
                elif op == 'exist':
                    current = dict(op='exist')
                elif op == 'count':
                    if block_id == len(clevr_program) - 1:
                        current = dict(op='count')
                elif op == 'equal_integer':
                    current = dict(op='count_equal')
                elif op == 'less_than':
                    current = dict(op='count_less')
                elif op == 'greater_than':
                    current = dict(op='count_greater')
                else:
                    raise ValueError('Unknown CLEVR operation: {}.'.format(op))

            if current is None:
                assert len(block['inputs']) == 1
                mapping[block_id] = mapping[block['inputs'][0]]
            else:
                current['inputs'] = list(map(mapping.get, block['inputs']))
                nscl_program.append(current)
                mapping[block_id] = len(nscl_program) - 1

        return nscl_program

    def nscl2program(self, nscl_program):
        programs = []
        for np in nscl_program:
            op = np["op"]
            if op == "filter":
                p = programs[np["inputs"][0]]
                for c in np['concept']:
                    p = ('Filter', p, [self.entry2idx_[c]])
            elif op == "query":
                p = ('Query', ('Unique', programs[np["inputs"][0]]),
                [[self.entry2idx_[_] for _ in self.concept_maps_[np["attribute"]]]])
            else:
                p = [camel_case(op)]
                for i in np['inputs']:
                    if 'relational_concept' in np or 'attribute' in np:
                        p.append(('Unique', programs[i]))
                    else:
                        p.append((programs[i]))
                if 'relational_concept' in np:
                    p.append([self.entry2idx_[np['relational_concept'][0]]])
                if 'attribute' in np:
                    p.append([self.entry2idx_[np['attribute']]])
                p = tuple(p)
            programs.append(p)
        return programs[-1]

    def metaconcept_text(self, supports, relations, concept_index):
        raise NotImplementedError

    def encode_text(self, question):
        encoded = self.word_vocab(question)
        length = len(encoded)
        assert length < self.max_question_length
        encoded.extend([self.word_vocab.pad] * (self.max_question_length - length))
        return encoded, length

    def encode_unknown(self, text, concept_id):
        for word in self.synonyms.get(self.names[concept_id], [self.names[concept_id]]):
            text = re.subn(f"(?<!\w){word}s?(\\b)", "unk\\1", text)[0]
        encoded = self.word_vocab(text)
        length = len(encoded)
        assert length < self.max_question_length
        encoded.extend([self.word_vocab.pad] * (self.max_question_length - length))
        return encoded, length

class BuilderDataset(Dataset):
    MAC_ROOT = "/data/vision/billf/scratch/jerrymei/BoxEmbedding/vendor/mac-network-original"
    texts = ["statement"]
    num_inputs = {}
    num_value_inputs = {}
    max_program_length = 50
    max_samekinds = 8
    metaconcepts_ = ['hypernym', 'hyponym', 'samekind']

    @property
    def info(self):
        return {**super(BuilderDataset, self).info, 'word_entries': len(self.word_vocab),
            'program_entries': len(self.program_vocab), 'start_id': self.program_vocab.start,
            'end_id': self.program_vocab.end, 'max_length': self.max_program_length}

    @property
    def principal_metric(self):
        return 'accuracy'

    def __init__(self, cfg, args):
        super().__init__(cfg, args)
        self.questions = []
        self.program_vocab = None
        self.mode = "build" if self.split != 'all' else 'inference'

    def _build_program_tokens(self):
        raise NotImplementedError

    def encode_program(self, program):
        encoded = [[self.program_vocab.start, self.program_vocab.start]]

        def _build_subtree(node):
            t = node['type']
            arg = self.program_vocab.unk if self.num_value_inputs[t] == 0 else self.program_vocab[
                node['value_inputs'][0]]
            entry = [self.program_vocab[t], arg]

            encoded.append(entry)
            for i in node['inputs']:
                _build_subtree(program[i])

        _build_subtree(program[-1])
        encoded.append([self.program_vocab.end, self.program_vocab.end])
        length = len(encoded)

        encoded.extend([[self.program_vocab.pad, self.program_vocab.pad]] * (self.max_program_length - length))
        assert length < self.max_program_length
        return encoded, length

    def encode_metaconcept(self, program):
        encoded = [[self.program_vocab.start, self.program_vocab.start]]
        for p in program:
            encoded.append([self.program_vocab[p['type']], self.program_vocab[p['value_inputs'][0]]])
        encoded.append([self.program_vocab.end, self.program_vocab.end])
        length = len(encoded)

        assert length < self.max_program_length
        encoded.extend([[self.program_vocab.pad, self.program_vocab.pad]] * (self.max_program_length - length))
        return encoded, length

    def decode_program(self, predicted):
        def canonical(operation):
            return operation.replace('_material', '').replace('_size', '').replace('_color', '').replace(
                '_shape', '')

        es = nonzero(predicted[..., 0] == self.program_vocab.end)
        end_index = es[0] if len(es) > 0 else predicted.shape[0]
        program_encoded = predicted[1:end_index].tolist()
        family = tuple(canonical(self.program_vocab.index2word[t]) for t, a in program_encoded)
        program = []

        def expand_node():
            if len(program_encoded) == 0:
                program.append({'type': 'scene', 'value_inputs': [], 'inputs': []})
                return len(program) - 1
            t, a = [self.program_vocab.index2word[_] for _ in program_encoded.pop(0)]
            value_inputs = [a] if a in self.named_entries_ else [] if self.num_value_inputs[t] == 0 else [
                random.choice(self.named_entries_)]
            p = {'type': t, 'value_inputs': value_inputs}
            ii = [expand_node() for _ in range(self.num_inputs[t])]
            p['inputs'] = ii
            program.append(p)
            return len(program) - 1

        expand_node()
        return program, family

    def decode_metaconcept(self, predicted):
        end_index = nonzero(predicted[..., 0] == self.program_vocab.end)[0]
        program_encoded = predicted[1:end_index].tolist()
        family = tuple(self.program_vocab.index2word[t] for t, a in program_encoded)
        program = [
            {'type': self.program_vocab.index2word[t], 'value_inputs': [self.program_vocab.index2word[a]],
                'inputs': []} for t, a in program_encoded if a not in self.program_vocab.special_tokens]
        return program, family

    def encode(self, text, program, name):
        encoded, length = self.encode_text(text)
        target, _ = self.encode_program(program) if name != 'metaconcept' else self.encode_metaconcept(program)
        return {f'{name}_encoded': encoded, f'{name}_length': length, f'{name}_target': target, f'{name}': text,
            f"{name}_program": program}

    def batch_build_handler(self, inputs, outputs):
        accuracies = {}
        for t in ["statement", "metaconcept", "question"]:
            # noinspection PyTypeChecker
            accuracies[f"{t}_accuracy"] = torch.all(
                torch.all(inputs[f"{t}_target"] == outputs[f"{t}_predicted"], -1), -1).tolist()
        return accuracies

    def metric_build_handler(self, evaluated):
        metrics = {}
        for k, v in evaluated.items():
            if k != "mode":
                metrics[k] = torch.tensor(v).float().mean().item()
        metrics['accuracy'] = mean(metrics.values())
        return metrics

    def exist_question_program(self, candidate):
        return [{'type': 'scene', 'inputs': [], 'value_inputs': []},
            {'type': 'filter', 'inputs': [0], 'value_inputs': [self.named_entries_[candidate]]},
            {'type': 'exist', 'inputs': [1], 'value_inputs': []}]

    def exist_statement_program(self, candidate):
        return [{'type': 'scene', 'inputs': [], 'value_inputs': []}]

    def __getitem__(self, index):
        question_index = self.indices_split[index]
        question = {k: torch.tensor(v) if re.search('encoded|target', k) else v for k, v in
            self.questions[question_index].items()}
        return {**question, 'question_index': question_index, 'index': index, 'info': self.info}

    def filter_question_program(self, candidate, filters):
        return [{'type': 'scene', 'inputs': [], 'value_inputs': []},
            *({'type': 'filter', 'inputs': [i], 'value_inputs': [self.named_entries_[f]]} for i, f in
                enumerate(filters)),
            {'type': 'filter', 'inputs': [len(filters)], 'value_inputs': [self.named_entries_[candidate]]},
            {'type': 'exist', 'inputs': [len(filters) + 1], 'value_inputs': []}]

    def filter_statement_program(self, candidate, filters):
        return [{'type': 'scene', 'inputs': [], 'value_inputs': []},
            *({'type': 'filter', 'inputs': [i], 'value_inputs': [self.named_entries_[f]]} for i, f in
                enumerate(filters))]


import mlflow
from models.models import *
from preprocessing.data_preprocessing import *
from utils.utils import *

if __name__ == "__main__":
    #mlflow.set_tracking_uri("http://127.0.0.1:5000")
    mlflow.set_experiment("Zindi_experiment")
    train = pd.read_csv('../umojahack-africa-2022-beginner-challenge/train.csv', parse_dates=['Datetime'])
    inference = pd.read_csv('../umojahack-africa-2022-beginner-challenge/test.csv', parse_dates=['Datetime'])
    train = date_processing(train)
    X = train.drop('Offset_fault', axis='columns')
    y = train.iloc[:,5]
    X_train, X_test, y_train, y_test = train_test_split(X,y, random_state=42, test_size=0.2, stratify=y)
    X_train = outlier_processing(X_train)
    X_train = missing_values_processing(X_train)
    X_train = X_train.iloc[:,1:]
    X_test = X_test.iloc[:,1:]
    X_test['output'] = y_test
    X_test.dropna(inplace= True)
    y_test = X_test['output']
    X_test = X_test.iloc[:,:-1]
    with mlflow.start_run():
        classifier, train_accuracy, test_accuracy = svm_classifier(X_train, X_test, y_train, y_test)
        y_pred = classifier.predict(y_test)
        f1, auc = calculate_metrics(classifier, X_test, y_test, y_pred)
        params = {"kernel":'rbf', "random_state": 42}
        metrics = {"train_accuracy": train_accuracy, "test_accuracy": test_accuracy,
                "f1": f1, "auc": auc}
        roc = plot_auc_roc(y_test, y_pred)
        conf_matrix = confusion_matrix(classifier, X_test, y_test)
        mlflow.log_params(params)
        mlflow.log_metrics(metrics)
        mlflow.log_figure(roc, "svm_roc.png")
        mlflow.log_figure(conf_matrix, "svm_confusion_matrix.png")
        mlflow.sklearn.log_model(classifier, artifact_path="models")
        mlflow.register_model(classifier, "SVC")

    with mlflow.start_run():
        clf, train_accuracy, test_accuracy = lr_classifier(X_train, X_test, y_train, y_test)
        y_pred = clf.predict(y_test)
        f1, auc = calculate_metrics(clf, X_test, y_test, y_pred)
        params = {"random_state": 42, "C": 1e5}
        metrics = {"train_accuracy": train_accuracy, "test_accuracy": test_accuracy,
                "f1": f1, "auc": auc}
        roc = plot_auc_roc(y_test, y_pred)
        conf_matrix = confusion_matrix(clf, X_test, y_test)
        mlflow.log_params(params)
        mlflow.log_metrics(metrics)
        mlflow.log_figure(roc, "lr_roc.png")
        mlflow.log_figure(conf_matrix, "lr_confusion_matrix.png")
        mlflow.sklearn.log_model(clf, artifact_path="models")
        mlflow.register_model(clf, "LogisticRegression")

    with mlflow.start_run():
        tree_clf, train_accuracy, test_accuracy = tree_classifier(X_train, X_test, y_train, y_test)
        y_pred = tree_clf.predict(y_test)
        f1, auc = calculate_metrics(tree_clf, X_test, y_test, y_pred)
        params = {"max_depth": 10}
        metrics = {"train_accuracy": train_accuracy, "test_accuracy": test_accuracy,
                "f1": f1, "auc": auc}
        roc = plot_auc_roc(y_test, y_pred)
        conf_matrix = confusion_matrix(tree_clf, X_test, y_test)
        mlflow.log_params(params)
        mlflow.log_metrics(metrics)
        mlflow.log_figure(roc, "tree_roc.png")
        mlflow.log_figure(conf_matrix, "tree_confusion_matrix.png")
        mlflow.sklearn.log_model(tree_clf, artifact_path="models")
        mlflow.register_model(tree_clf, "decision_trees")

    with mlflow.start_run():
        for_clf, train_accuracy, test_accuracy = forest_classifier(X_train, X_test, y_train, y_test)
        y_pred = for_clf.predict(y_test)
        f1, auc = calculate_metrics(for_clf, X_test, y_test, y_pred)
        metrics = {"train_accuracy": train_accuracy, "test_accuracy": test_accuracy,
                "f1": f1, "auc": auc}
        roc = plot_auc_roc(y_test, y_pred)
        conf_matrix = confusion_matrix(for_clf, X_test, y_test)
        mlflow.log_metrics(metrics)
        mlflow.log_figure(roc, "random_forest_roc.png")
        mlflow.log_figure(conf_matrix, "random_forest_confusion_matrix.png")
        mlflow.sklearn.log_model(for_clf, artifact_path="models")
        mlflow.register_model(for_clf, "random_forest")

    with mlflow.start_run():
        sgd_clf, train_accuracy, test_accuracy = sgd_classifier(X_train, X_test, y_train, y_test)
        y_pred = sgd_clf.predict(y_test)
        f1, auc = calculate_metrics(sgd_clf, X_test, y_test, y_pred)
        params = {"max_iter": 1000, "tol": 1e-3}
        metrics = {"train_accuracy": train_accuracy, "test_accuracy": test_accuracy,
                "f1": f1, "auc": auc}
        roc = plot_auc_roc(y_test, y_pred)
        conf_matrix = confusion_matrix(sgd_clf, X_test, y_test)
        mlflow.log_params(params)
        mlflow.log_metrics(metrics)
        mlflow.log_figure(roc, "sgd_roc.png")
        mlflow.log_figure(conf_matrix, "sgd_confusion_matrix.png")
        mlflow.sklearn.log_model(sgd_clf, artifact_path="models")
        mlflow.register_model(sgd_clf, "sgd")


# from django.shortcuts import render
# # Create your views here.

from rest_framework.decorators import api_view, permission_classes  #ls5
from rest_framework.response import Response
from products.serializers import ProductSerializer, ProductValidateSerializer  #ls4
from products.serializers import BrandSerializer, ReviewSerializer  #ls6
from products.models import Product, Review, Brand  #ls6
from rest_framework import status
from rest_framework.permissions import IsAuthenticated  #ls5
from rest_framework.permissions import AllowAny, IsAdminUser  #ls6
from rest_framework.generics import ListAPIView, ListCreateAPIView, RetrieveUpdateDestroyAPIView  #ls6
from rest_framework.pagination import PageNumberPagination  #ls6
from rest_framework.viewsets import ModelViewSet #ls6

@api_view(['GET'])
def test(request):
    data = {
        'text': 'Hello World!',
        'int': 100,
        'float': 2.99,
        'list': [1, 2, 3, 4],
        'bool': False
    }
    return Response(data=data)

#ls3
@api_view(['POST'])
def test_post(request):
    print(request.data['name'])
    print(request.data.get('name')) #--более предпочтит использовать
    return Response(data={'message': 'OK'})

# @api_view(['GET']) #ls2
# def product_list_view(request):
#     products = Product.objects.all()
#     serializer = ProductSerializer(products, many=True)
#     return Response(data=serializer.data)

@api_view(['GET', 'POST']) #ls3
@permission_classes([IsAuthenticated]) #ls5
def product_list_create_view(request):
    print(request.user) #ls5
    if request.method == 'GET':
        products = Product.objects.all()
        serializer = ProductSerializer(products, many=True)
        return Response(data=serializer.data)
    elif request.method == 'POST':
       serializer = ProductValidateSerializer(data=request.data) #ls4
       if not serializer.is_valid():
           return Response(status=status.HTTP_406_NOT_ACCEPTABLE,
                           data={'errors' : serializer.errors})
       # title = request.data.get('title')
       title = serializer.validated_data.get('title')
       # if title is None: #ls4
       #     title = ''
       # if len(title) < 2: #ls4
       #     return Response(status=status.HTTP_406_NOT_ACCEPTABLE,
       #                     data={'error': 'title length less than 2'})
       weight = request.data.get('weight')
       price = request.data.get('price')
       is_stock = request.data.get('is_stock')
       valid_until = request.data.get('valid_until')
       brand_id = request.data.get('brand_id')
       # product = Product.objects.create(**serializer.validated_data)
       product = Product.objects.create(title = title, weight = weight,
                                        price = price, is_stock= is_stock,
                                        valid_until = valid_until,
                                        brand_id = brand_id)
       for i in serializer.validated_data['reviews']:
           Review.objects.create(product=product, text=i['text'], stars=i['stars'])
       return Response(data=ProductSerializer(product).data)

# @api_view(['GET']) #ls2
# def product_item_view(request, id):
#     try:
#         product = Product.objects.get(id=id)
#     except Product.DoesNotExist:
#         return Response(data={'error': 'Product not Found!!!'},
#                         status=status.HTTP_404_NOT_FOUND)
#                         # status=404)
#     serializer = ProductSerializer(product)
#     return Response(data=serializer.data)

#ls3
@api_view(['GET', 'PUT', 'DELETE'])
def product_item_view(request, id):
    try:
        product = Product.objects.get(id=id)
    except Product.DoesNotExist:
        return Response(data={'error': 'Product not Found!!!'},
                        status=status.HTTP_404_NOT_FOUND)
                        # status=404)
    if request.method == 'GET':
        serializer = ProductSerializer(product)
        return Response(data=serializer.data)
    elif request.method == 'DELETE':
        product.delete()
        return Response(data={'message': 'Product removed'})
    else:
        product.title = request.data.get('title')
        product.weight = request.data.get('weight')
        product.price = request.data.get('price')
        product.is_stock = request.data.get('is_stock')
        product.valid_until = request.data.get('valid_until')
        product.brand_id = request.data.get('brand id')
        product.save()
        return Response(data=ProductSerializer(product).data)

#ls6
# class BrandListAPIView(ListAPIView):
class BrandListAPIView(ListCreateAPIView):
    queryset = Brand.objects.all()
    serializer_class = BrandSerializer
    # permission_classes = IsAuthenticated,
    pagination_class = PageNumberPagination

class BrandItemAPIView(RetrieveUpdateDestroyAPIView):
    queryset = Brand.objects.all()
    serializer_class = BrandSerializer
    # lookup_field = 'id'

class ReviewModelViewSet(ModelViewSet):
    queryset = Review.objects.all()
    serializer_class = ReviewSerializer


import hashlib

from django.test import TestCase

from ..models import DimUser, Simulation, SimulationGroup, SimulationInputFile

class SimInputFileTests(TestCase):
    """
    Tests for the data model for simulation input files.
    """

    @classmethod
    def setUpClass(cls):
        cls.user = DimUser.objects.create(username='test-user')
        cls.sim_group = SimulationGroup.objects.create(submitted_by=cls.user)
        cls.simulation = Simulation.objects.create(group=cls.sim_group)

    @classmethod
    def tearDownClass(cls):
        for data_model in Simulation, SimulationGroup, DimUser:
            data_model.objects.all().delete()

    def assertContentsEqual(self, input_file, expected_contents):
        self.assertIsInstance(input_file, SimulationInputFile)
        retrieved_contents = input_file.get_contents()
        self.assertEqual(retrieved_contents, expected_contents)
        expected_hash = hashlib.md5(expected_contents).hexdigest()
        self.assertEqual(input_file.metadata[input_file.MetadataKeys.CHECKSUM], expected_hash)

    def test_one_file(self):
        """
        Test with an input file associated with just one simulation.
        """
        contents = 'Four score and seven years ago, our forefathers ...'
        sim_input_file = SimulationInputFile.objects.create_file(contents, created_by=self.user)
        self.assertContentsEqual(sim_input_file, contents)

        sim_input_file.simulations.add(self.simulation.id)
        self.assertEqual(sim_input_file.simulations.count(), 1)
        self.assertEqual(sim_input_file.simulations.all()[0], self.simulation)

    def test_file_shared_by_many(self):
        """
        Test where an input file is shared by multiple simulations.
        """
        contents = "Roses are red, violets are blue, ..."
        sim_input_file = SimulationInputFile.objects.create_file(contents, created_by=self.user)
        self.assertContentsEqual(sim_input_file, contents)

        simulation2 = Simulation.objects.create(group=self.sim_group)
        simulation3 = Simulation.objects.create(group=self.sim_group)

        sim_input_file.simulations.add(self.simulation, simulation2, simulation3)
        self.assertEqual(sim_input_file.simulations.count(), 3)
        expected_pks = (self.simulation.id, simulation2.id, simulation3.id)
        for simulation in sim_input_file.simulations.all():
            self.assertIn(simulation.id, expected_pks)

    def test_foo(self):
        """
        Test where a simulation has multiple input files.
        """
        empty_contents = ''
        empty_input_file = SimulationInputFile.objects.create_file(empty_contents, created_by=self.user)
        self.assertContentsEqual(empty_input_file, empty_contents)
        empty_input_file.simulations.add(self.simulation)

        json_contents = '''
            {
                "foo_parameter": 42,
                "bar_parameter": "/path/to/data.csv"
            }
        '''
        json_input_file = SimulationInputFile.objects.create_file(json_contents, created_by=self.user)
        self.assertContentsEqual(json_input_file, json_contents)
        json_input_file.simulations.add(self.simulation)

        self.assertEqual(self.simulation.input_files.count(), 2)
        for input_file in (empty_input_file, json_input_file):
            self.assertIsInstance(self.simulation.input_files.get(id=input_file.id), SimulationInputFile)

    def test_open_for_reading(self):
        """
        Test the open_for_reading() method of an input file.
        """
        empty_om_scenario = """
            <?xml version='1.0' encoding='UTF-8' standalone='no'?>
            <om:scenario xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
                         name='name'
                        schemaVersion='32'
                        xsi:schemaLocation='http://openmalaria.org/schema/scenario_32 scenario_32.xsd'
                        xmlns:om='http://openmalaria.org/schema/scenario_32'
                        analysisNo='0'
                        wuID='0'>
            </om:scenario>"
        """
        sim_input_file = SimulationInputFile.objects.create_file(empty_om_scenario, created_by=self.user)
        with sim_input_file.open_for_reading() as f:
            retrieved_contents = f.read()
        self.assertEqual(retrieved_contents, empty_om_scenario)


# -*- coding: utf-8 -*-
import re
import math
import unicodedata

from collections import Counter

def strip_accents(s):
	return ''.join(c for c in unicodedata.normalize('NFKD', s)
		if unicodedata.category(c) != 'Mn')

# Returns a dial code string for a word.
## Omits special characters
def word_to_number(word):
	number = "";
	mapping_number = {
		".": 1,
		"a": 2,
		"b": 2,
		"c": 2,
		"d": 3,
		"e": 3,
		"f": 3,
		"g": 4,
		"h": 4,
		"i": 4,
		"j": 5,
		"k": 5,
		"l": 5,
		"m": 6,
		"n": 6,
		"ñ": 6,
		"o": 6,
		"p": 7,
		"q": 7,
		"r": 7,
		"s": 7,
		"t": 8,
		"u": 8,
		"v": 8,
		"w": 9,
		"x": 9,
		"y": 9,
		"z": 9};

	word = ''.join(e for e in word if e.isalnum())
	word = re.sub(r'[0-9]', '', word);
	word = word.replace('á', 'a');
	word = word.replace('é', 'e');
	word = word.replace('í', 'i');
	word = word.replace('ó', 'o');
	word = word.replace('ú', 'u');
	word = word.lower();

	for i in range (0, len(word)):
		character = word[i];

		if (character in mapping_number):
			number += str(mapping_number[character]);
		else:
			print('Character ' + character + ' not recognised');

	return number;

def sentence_to_numbers(sentence):
	out = ""
	for word in ' '.join(sentence.split()).split():
		out += word_to_number(word) + " "
	return out

def get_cosine(vec1, vec2):
	intersection = set(vec1.keys()) & set(vec2.keys())
	numerator = sum([vec1[x] * vec2[x] for x in intersection])

	sum1 = sum([vec1[x]**2 for x in vec1.keys()])
	sum2 = sum([vec2[x]**2 for x in vec2.keys()])
	denominator = math.sqrt(sum1) * math.sqrt(sum2)

	if not denominator:
		return 0.0
	else:
		return float(numerator) / denominator

def text_to_vector(text):
	word = re.compile(r'\w+')
	words = word.findall(text)
	return Counter(words)

def word_to_vector(word):
	char = re.compile(r'\w')
	chars = char.findall(word)
	return Counter(chars)


from fastapi import APIRouter, FastAPI
from users.views import router as users
from auth.views import router as auth
from database import create_database

app = FastAPI()
router = APIRouter()

@app.on_event("startup")
def on_starup():
    create_database()

@router.get('/')
def index():
    return {'status':'ok'}

router.include_router(users, prefix='/users', tags=['Users'])
router.include_router(auth, prefix='/auth', tags=['Authentication'])
app.include_router(router)


"""
Ce fichier sera appelé avec des paramètres par le serveur unified remote
"""

import argparse
import json

parser = argparse.ArgumentParser()
parser.add_argument('-r','--red', type=int, help='tell server to change the red color')
parser.add_argument('-g','--green', type=int, help='tell server to change the green color')
parser.add_argument('-b','--blue', type=int,default=10, help='tell server to change the blue color')
parser.add_argument('-p','--power', type=bool, help='tell server to turn the lights on (1) or off (0)')
args = parser.parse_args()

f = open('/home/guillaume/git_folder/Light_server/comm_file','w')
liste=[]
for arg in args.__dict__ :
  if args.__dict__[arg]!=None:
      liste.append([arg,args.__dict__[arg]])
f.write(json.dumps(liste))
f.close


from django.contrib.auth.forms import UserCreationForm, UserChangeForm
from django.forms import Textarea

from .models import CustomUser

class CustomUserCreationForm(UserCreationForm):

    class Meta:
        model = CustomUser
        fields = ('email',)
        widgets = {
            'email' : Textarea(attrs={'cols' : 80, 'rows': 20})
        }

class CustomUserChangeForm(UserChangeForm):

    class Meta:
        model = CustomUser
        fields = ('email',)
        widgets = {
            'email' : Textarea(attrs={'cols' : 80, 'rows': 20})
        }


# 실습 과제: IMDB 영화 정보 크로링 통해 csv로 저장하기

import requests
from bs4 import BeautifulSoup

f = open('test.csv','w',encoding='utf-8')
f.write('title,rate \n')
headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.83 Safari/537.36'}

url ='https://www.imdb.com/chart/boxoffice/?ref_=nv_ch_cht'
raw = requests.get(url,headers=headers)

soup =BeautifulSoup(raw.text,'html.parser')

# 중간 print로 확인해보는 작업 얘기해주기
# print(soup)
container = soup.select('tbody > tr')
for info in container:
    title = info.select_one('td.titleColumn > a').text.strip()
    rate = info.select_one('td.ratingColumn').text.strip()
    f.write(title + ',' + rate + '\n')

f.close()


from helpers import randfloat
from time import sleep
import keyboard as kb
import sys

# fn=input("File name: ")
fn=sys.argv[1]
waits = bool(int(sys.argv[2]))
with open(fn,"r") as file:
    print("reading "+fn)
    lines=file.readlines()
    print("start in 5 seconds!")
    sleep(5)
    for line in lines:
        if waits and .2>=randfloat(0.0,1.0):
            sleep(randfloat(0.25,.9))
        text=line[:line.index("\n")]
        kb.press_and_release("command+shift+left")#reset indents
        kb.write(text)
        print(text)
        kb.press_and_release("command+shift+down")#delete stray ) or ] autogenerated by IDE
        kb.press_and_release("enter")


from django.shortcuts import render
from django.http import JsonResponse
from tethys_sdk.gizmos import SelectInput, RangeSlider
from django.contrib.auth.decorators import login_required

from .options import *
from .utilities import *
from .app import Gfs as App
from .utilities import new_id

def home(request):
    """
    Controller for the app home page.
    """

    gfsdate = currentgfs()

    variables = SelectInput(
        display_text='Select GFS Variable',
        name='variables',
        multiple=False,
        original=True,
        options=gfs_variables(),
    )

    levels = SelectInput(
        display_text='Available Forecast Levels',
        name='levels',
        multiple=False,
        original=True,
        options=variable_levels()['al'],
    )

    regions = SelectInput(
        display_text='Pick A World Region (ESRI Living Atlas)',
        name='regions',
        multiple=False,
        original=True,
        options=list(worldregions())
    )

    colorscheme = SelectInput(
        display_text='GFS Color Scheme',
        name='colorscheme',
        multiple=False,
        original=True,
        options=wms_colors(),
        initial='rainbow'
    )

    opacity = RangeSlider(
        display_text='GFS Layer Opacity',
        name='opacity',
        min=.5,
        max=1,
        step=.05,
        initial=1,
    )

    gj_color = SelectInput(
        display_text='Boundary Border Colors',
        name='gjClr',
        multiple=False,
        original=True,
        options=geojson_colors(),
        initial='#ffffff'
    )

    gj_opacity = RangeSlider(
        display_text='Boundary Border Opacity',
        name='gjOp',
        min=0,
        max=1,
        step=.1,
        initial=1,
    )

    gj_weight = RangeSlider(
        display_text='Boundary Border Thickness',
        name='gjWt',
        min=1,
        max=5,
        step=1,
        initial=2,
    )

    gj_fillcolor = SelectInput(
        display_text='Boundary Fill Color',
        name='gjFlClr',
        multiple=False,
        original=True,
        options=geojson_colors(),
        initial='rgb(0,0,0,0)'
    )

    gj_fillopacity = RangeSlider(
        display_text='Boundary Fill Opacity',
        name='gjFlOp',
        min=0,
        max=1,
        step=.1,
        initial=.5,
    )

    context = {
        # data options
        'variables': variables,
        'levels': levels,
        'gfsdate': gfsdate,
        'regions': regions,

        # display options
        'colorscheme': colorscheme,
        'opacity': opacity,
        'gjClr': gj_color,
        'gjOp': gj_opacity,
        'gjWt': gj_weight,
        'gjFlClr': gj_fillcolor,
        'gjFlOp': gj_fillopacity,

        # metadata
        'app': App.package,
        'githublink': App.githublink,
        'datawebsite': App.datawebsite,
        'version': App.version,
        'thredds_url': App.get_custom_setting('thredds_url'),
        'instance_id': new_id(),
    }

    return render(request, 'gfs/home.html', context)

@login_required()
def checkworkflowstatus(request):
    threddspath = os.path.join(App.get_custom_setting('thredds_path'))
    fail = os.path.join(threddspath, 'last_run_failed.txt')
    run = os.path.join(threddspath, 'running.txt')
    last = os.path.join(threddspath, 'last_run.txt')
    if os.path.isfile(last):
        with open(last, 'r') as f:
            return JsonResponse({'succeeded': f.readlines()})
    elif os.path.isfile(run):
        return JsonResponse({'running': 'running'})
    elif os.path.isfile(fail):
        with open(fail, 'r') as f:
            return JsonResponse({'failed': f.readlines()})
    else:
        return JsonResponse({'unknown': 'no indicator files were found'})


from PyQt5 import QtWidgets, QtGui, QtCore
from PyQt5.QtCore import QThread
from GUI import Ui_MainWindow
import sys
import speech_recognition as sr

class TreadVoice(QThread):

    def __init__(self):
        QThread.__init__(self)

    def __del__(self):
        self.wait()

    def run(self):
        pass

class mainwindow(QtWidgets.QMainWindow):

    def __init__(self):
        super(mainwindow, self).__init__()
        self.ui = Ui_MainWindow()
        self.ui.setupUi(self)

        self.ui.pushButton_listen.clicked.connect(self.voiceListen)

    def voiceListen(self):
        print('Произошёл запуск')
        r = sr.Recognizer()

        with sr.Microphone() as source:
            # self.ui.label.setText("Говорите")
            r.pause_threshold = 0.5
            r.adjust_for_ambient_noise(source, duration=1)
            audio = r.listen(source)

        try:
            task = r.recognize_google(audio, language="ru-RU").lower()
            if "закрыть" in task:
                sys.exit(0)
            else:
                self.ui.listWidget.addItem("Вы сказали " + task)
                # self.ui.label.setText("Вы сказали " + task)
                # self.ui.label.adjustSize()
        except (sr.UnknownValueError, sr.RequestError):
            self.ui.listWidget.addItem('Не понятно, повторите')
            # self.ui.label.setText('Не понятно, повторите')
            task = self.voiceListen()
        return task

    # def sayText(task):
    #     if "стоп" in task:
    #         sys.exit(0)
    #     else:
    #         self.ui.label.setText("Вы сказали " + task)
    #         self.ui.label.adjustSize()

    def main(self):
        self.voiceListen()

app = QtWidgets.QApplication([])
application = mainwindow()
application.show()

sys.exit(app.exec())


import tweepy as tw
import datetime, random, smtplib
from datetime import timedelta

auth = tw.OAuthHandler('consumer_key','consumer_secret') #insert consumer_key and consumer_secret
auth.set_access_token('access_token','access_token_secret') #insert access_token and access_token_secret
api = tw.API(auth, wait_on_rate_limit=True)

user = 'naval' #insert the username for whose random favorite tweet you'd like to get over email
all_fav_tweets = []
old_date = datetime.datetime(2019,1,1,0,0,0)
tweets=api.favorites(user,tweet_mode='extended',include_entities=True)

sender_email = "a@gmail.com"  # Enter your address
receiver_email = "d@gmail.com"  # Enter receiver address

for t in tweets:
    all_fav_tweets.append(t._json['id'])
print(f"Fetched {len(all_fav_tweets)} tweets, fetching more")

while (tweets[-1].created_at>old_date and len(all_fav_tweets)<=1000):
    tweets=api.favorites(user,max_id=tweets[-1].id,tweet_mode='extended',include_entities=True)
    for t in tweets:

        all_fav_tweets.append(t._json['id'])
    print(f"Fetched {len(all_fav_tweets)} tweets, fetching more")
random_tweet_id=all_fav_tweets[random.randint(0,len(all_fav_tweets))]
random_tweet=api.get_status(random_tweet_id,tweet_mode='extended',include_entities=True)
random_tweet_text=random_tweet._json['full_text']
random_tweet_user=random_tweet._json['user']['screen_name']
random_tweet_url = "https://twitter.com/"+str(random_tweet_user)+"/status/"+str(random_tweet_id)

SUBJECT = "Random Liked Tweet For The Day"
TEXT = "\nTweet By "
TEXT = TEXT + str(random_tweet_user) + ": " +str(random_tweet_text) + "\n\n" + str(random_tweet_url)
message = 'Subject: {}\n\n{}'.format(SUBJECT, TEXT)
print(message)
server = smtplib.SMTP_SSL('smtp.gmail.com', 465)
server.login("email id", "password") #email id and password

message= message.encode('ascii', 'ignore').decode('ascii')
server.sendmail(sender_email,receiver_email,message)
print("Email Sent")
server.quit()


import argparse
import os

def parse_ko_metadata(metadata_f):
    """ Parses the ko metadata file and returns a list of KOs """

    kos = []
    with open(metadata_f, 'r') as IN:
        for line in IN:
            # skip header
            if line.startswith("Trait\t"):
                continue

            kos.append(line.split("\t", 1)[0])

    return sorted(kos)

def count_KOs(ko_table_f, ko_list):
    """ Parses a JGI KO table and counts KOs on the list """
    ko_counts = {k: 0 for k in ko_list}
    ko_not_in_list = 0
    with open(ko_table_f, 'r') as IN:
        for line in IN:
            # skip header
            if line.startswith("KO ID"):
                continue

            elems = line.rstrip().split("\t")

            ko = elems[0]
            try:
                count = elems[-1]
            except IndexError:
                print((line,))
                sys.exit()

            # strip the "KO:" off the ko name
            ko = ko[3:]

            try:
                ko_counts[ko] = float(count)
            except KeyError:
                ko_not_in_list += 1

    print("{} KOs not in the ko_list.".format(ko_not_in_list))
    return ko_counts

def main(args):
    ko_list = parse_ko_metadata(args.ko_metadata)

    with open(args.out, 'w') as OUT:
        OUT.write("\t".join(["OTU_IDs"] + ko_list) + "\n")

        for ko_table in args.ko:
            name = os.path.splitext(os.path.basename(ko_table))[0]

            ko_counts = count_KOs(ko_table, ko_list)

            # count the number of
            ko_not_counted = list(ko_counts.values()).count(0)

            print("{} KOs not found in genome.".format(ko_not_counted))
            print("{} KOs found in the genome.".format(len(list(ko_counts.values())) - ko_not_counted))
            print("")

            to_write = [name]
            [to_write.append(str(ko_counts[ko])) for ko in ko_list]
            OUT.write("\t".join(to_write) + "\n")

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Converts JGI's KO table into a PICRUSt trait table")
    parser.add_argument("-ko", help="one or more JGI KO tables", nargs="+")
    parser.add_argument("-ko_metadata", help="the KO metadata table from the PICRUSt deconstructed files", required=True)
    parser.add_argument("-out", help="output path for the trait table", default="trait_table.tab")

    args = parser.parse_args()

    main(args)


#coding:UTF-8
import os
import urllib

from datetime import datetime
import locale

import jinja2
import webapp2

JINJA_ENVIRONMENT = jinja2.Environment(
    loader=jinja2.FileSystemLoader(os.path.dirname(__file__)),
    extensions=['jinja2.ext.autoescape'],
    autoescape=True)

class MainPage(webapp2.RequestHandler):

	def get(self):

		html_template = "index.html"

		d = datetime.today()
		time_str = u'%s時%s分%s.%s秒' % (d.hour, d.minute, d.second, d.microsecond)
		template_values = {
			'time_str': time_str
		}

		t = JINJA_ENVIRONMENT.get_template("dtime.html")
		self.response.write(t.render(template_values))

app = webapp2.WSGIApplication([
    ('/.*', MainPage),
], debug=True)


# -*- coding: utf-8 -*-
# @Time    : 2020/6/7 9:29
# @Author  : Equator
import matplotlib.pyplot as plt
import code.preprocessing.data_scan as data_scan
from pandas.plotting import scatter_matrix
import numpy as np

# 单一特征图表
def hist(data):
    data.hist(sharex=False, sharey=False, xlabelsize=1, ylabelsize=1)
    plt.show()

# 密度图
def density(data):
    data.plot(kind='density', subplots=True, layout=(4, 4), sharex=False, fontsize=1)
    plt.show()

# 箱线图
def box(data):
    data.plot(kind='box', subplots=True, layout=(4, 4), sharex=False, fontsize=1)
    plt.show()

# 散点图矩阵
def scatter(data):
    scatter_matrix(data, figsize=(48, 48))
    plt.show()

def figure(data):
    fig = plt.figure(figsize=(13, 13))
    ax = fig.add_subplot(111)
    cax = ax.matshow(data.corr(), vmin=-1, vmax=1, interpolation='none')
    fig.colorbar(cax)
    # 刻度
    ticks = np.arange(0, 14, 1)
    ax.set_xticks(ticks)
    ax.set_yticks(ticks)
    names = list(data.columns)
    ax.set_xticklabels(names)
    ax.set_yticklabels(names)
    plt.show()

if __name__ == '__main__':
    data = data_scan.get_data()
    # hist(data)
    # density(data)
    # box(data)
    # scatter(data)
    figure(data)


import grpc
import test_pb2
import test_pb2_grpc
import datetime
import argparse
import threading
import time
import logging

logging.basicConfig(format='%(asctime)s %(message)s',level=logging.INFO)

class heartbeatThread(threading.Thread):
    def __init__(self, stub, clientName):
        threading.Thread.__init__(self)
        self.stub = stub
        self.clientName = clientName

    def run(self):
        logging.info("开始发送心跳包")
        m= self.stub.heartbeat(self.heartbeatGenerator())
        while True:
            logging.debug("收到心跳包响应:%s"%m.next())

    def heartbeatGenerator(self):
        while True:
            yield test_pb2.heartbeatReq(name=self.clientName)
            time.sleep(2)

def ChannelMonitor(ChannelConnectivity):
    print(ChannelConnectivity)

def run(name):
    time.sleep(3)#解决客户端在半连接周期内重连的问题...每次都等待超过一个半连接周期后再进行链接

    channel = grpc.insecure_channel("47.95.205.174:50051")

    channel.subscribe(ChannelMonitor)

    stub = test_pb2_grpc.pushServerStub(channel)

    responser = stub.subscribe(test_pb2.clientInfo(name=name))
    logging.info("订阅成功!%s"%responser.next())

    heartbeatThread(stub, name).start()

    while True:
        print(responser.next().info)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Process some integers.')
    parser = argparse.ArgumentParser(description='Description of your program')
    parser.add_argument('-n', '--name', help='name of this client',
                        required=True, type=str)
    args = vars(parser.parse_args())
    name = args['name']
    run(name)


import networkx as nx
import matplotlib.pyplot as plt
from sklearn.manifold import TSNE

def visualize_graph(G, color):
    plt.figure(figsize=(7,7))
    plt.xticks([])
    plt.yticks([])
    nx.draw_networkx(G, pos=nx.spring_layout(G, seed=42), with_labels=False,
                     node_color=color, cmap="Set2")
    plt.show()

def visualize_embedding(h, color, epoch=None, loss=None):
    plt.figure(figsize=(7,7))
    plt.xticks([])
    plt.yticks([])
    h = h.detach().cpu().numpy()
    plt.scatter(h[:, 0], h[:, 1], s=140, c=color, cmap="Set2")
    if epoch is not None and loss is not None:
        plt.xlabel(f'Epoch: {epoch}, Loss: {loss.item():.4f}', fontsize=16)
    plt.show()

def visualize_tsne(h, color):
    z = TSNE(n_components=2).fit_transform(h.detach().cpu().numpy())

    plt.figure(figsize=(10,10))
    plt.xticks([])
    plt.yticks([])

    plt.scatter(z[:, 0], z[:, 1], s=70, c=color, cmap="Set2")
    plt.show()


import sys
import json
import schedule
import time
import threading

reload(sys)
sys.setdefaultencoding('UTF8')

config_file = "load.conf"

instance_required_keys = ["instance_name", "schedule_pattern", "delay", "hive_table"]
config_required_keys = ["schedule_patterns", "instances_config"]
pattern_required_keys = ["mask"]

def write_config(config):
    with open(config_file, "w") as f:
        json.dump(config, f, indent=5)
        print("finished dump dic to file")

def load_config():
    with open(config_file, "r") as f:
        config = json.load(f)
        print("finished load config from json")
        return config

def validate_config(config_map, required_fields):
    for config_key in required_fields:
        if not config_map.has_key(config_key):
            print("Config key %s is required" % str(config_key))
            return False
    return True

def pattern_match(config_pattern, mask):
    return int("0b" + config_pattern, 2) & int("0b" + mask, 2) > 0

def schedule_job(instance_config, job_type):
    instance_name = instance_config.get(instance_required_keys[0])
    print("Active job type %s for instance %s" % (job_type, instance_name))
    if job_type == "secondly":
        schedule.every().second.do(job, instance_config, job_type)
    elif job_type == "minutely":
        schedule.every().minute.do(job, instance_config, job_type)
    elif job_type == "hourly":
        schedule.every().hour.do(job, instance_config, job_type)
    elif job_type == "daily":
        schedule.every().day.do(job, instance_config, job_type)
    elif job_type == "weekly":
        schedule.every(7).days.do(job, instance_config, job_type)
    else:
        print("Currently do not support schedule job type: %s" % job_type)

def start_schedule_job(instance_config, schedule_pattern):
    if not validate_config(instance_config, instance_required_keys):
        print("Instance config not passed validate: %s" % str(instance_config))
        return
    instance_pattern_config = instance_config.get(instance_required_keys[1])
    for pattern_type, pattern in schedule_pattern.items():
        if not validate_config(pattern, pattern_required_keys):
            print("Pattern config not passed validate: %s" % str(pattern))
        if pattern_match(instance_pattern_config, pattern.get(pattern_required_keys[0])):
            schedule_job(instance_config, pattern_type)

def job(instance_config, job_type):
    threading.Thread(target=job_task, args=(instance_config, job_type)).start()

def job_task(instance_config, job_type):
    instance_name = instance_config.get(instance_required_keys[0])
    print("Running job type %s on config %s" % (job_type, instance_name))

def run_main():
    config = load_config()
    if not validate_config(config, config_required_keys):
        print("Config not passed validate, please check config: %s" % str(config))
        return

    for instance_config in config.get(config_required_keys[1]):
        start_schedule_job(instance_config, config.get(config_required_keys[0]))

    while True:
        schedule.run_pending()
        time.sleep(1)

if __name__ == '__main__':
    run_main()


from django.db import models
from django.conf import settings
from django.db.models.signals import post_save
from django.dispatch import receiver
from institutions.models import InstitutionVerification
from submissions.models import *
from users.models import NewUser
from .pusher import pusher_client

class Notification(models.Model):
    class UnreadNofif(models.Manager):
        def get_queryset(self):
            return super().get_queryset().filter(isRead=False)

    options = (("verification", "verification"), ("submission", "submission"), ("recommendation", "recommendation"))
    type = models.CharField(choices=options, max_length=20)
    receiver = models.ManyToManyField(
        settings.AUTH_USER_MODEL,
    )
    sender = models.ForeignKey(
        settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name="+", blank=True, null=True
    )
    message = models.TextField(null=True, blank=True)
    isRead = models.BooleanField(default=False)
    dateCreated = models.DateTimeField(auto_now_add=True)
    dateModified = models.DateTimeField(auto_now=True)
    redirectID = models.IntegerField(null=True, blank=True)

    objects = models.Manager()
    unread = UnreadNofif()

    class Meta:
        ordering = ["-dateModified"]

    def __str__(self) -> str:
        return self.message

@receiver(post_save, sender=InstitutionVerification)
def apply_verification_handler(created, instance, *args, **kwargs):
    # if created:  # if a verification is created
    #     notif = Notification.objects.create(sender=instance.institution.owner, type="verification")

    #     notif.receiver.add(NewUser.objects.get(pk=1).id)
    #     notif.save()
    print(instance.status)
    if instance.status == "approved":

        pusher_client.trigger(
            "notification",
            instance.institution.creator.username,
            {
                "type": "verification",
                "message": "%s verification application is disapproved!" % instance.institution.name,
                "redirectID": instance.institution.id,
                # "room": self.request.data["room"],
            },
        )
        notif = Notification.objects.create(
            sender=NewUser.objects.get(pk=1),
            type="verification",
            message="%s verification application is disapproved!" % instance.institution.name,
            redirectID=instance.institution.id,
        )

        notif.receiver.add(instance.institution.creator)
        notif.save()
    if instance.status == "disapproved":

        pusher_client.trigger(
            "notification",
            instance.institution.creator.username,
            {
                "type": "verification",
                "message": "%s verification application is disapproved!" % instance.institution.name,
                "redirectID": instance.institution.id,
                # "room": self.request.data["room"],
            },
        )
        notif = Notification.objects.create(
            sender=NewUser.objects.get(pk=1),
            type="verification",
            message="%s verification application is disapproved!" % instance.institution.name,
            redirectID=instance.institution.id,
        )

        notif.receiver.add(instance.institution.creator)
        notif.save()

@receiver(post_save, sender=Submission)
def submission_created_handler(created, instance, *args, **kwargs):
    if created:
        # breakpoint()
        pusher_client.trigger(
            "notification",
            instance.file.folder.workspace.classroom.adviserInstance.username,
            {
                "type": "submission",
                "message": "A submission is created at %s" % instance.file.folder.workspace.classroom.name,
                "redirectID": instance.file.folder.workspace.classroom.id,
                # "room": self.request.data["room"],
            },
        )
        notif = Notification.objects.create(
            # sender=NewUser.objects.get(pk=1),
            type="submission",
            message="A submission is created at %s" % instance.file.folder.workspace.classroom.name,
            redirectID=instance.file.folder.workspace.classroom.id,
        )
        # adviserInstance
        notif.receiver.add(instance.file.folder.workspace.classroom.adviserInstance)
        notif.save()
        pass

@receiver(post_save, sender=Recommendation)
def recommendation_created_handler(created, instance, *args, **kwargs):
    if created:

        pusher_client.trigger(
            "notification",
            instance.department.institution.creator.username,
            {
                "type": "recommendation",
                "message": "A recommendation is created at %s" % instance.department.name,
                "redirectID": instance.department.id,
                # "room": self.request.data["room"],
            },
        )
        notif = Notification.objects.create(
            # sender=NewUser.objects.get(pk=1),
            type="recommendation",
            message="A recommendation is created at %s" % instance.department.name,
            redirectID=instance.department.id,
        )
        notif.receiver.add(instance.department.institution.creator)
        notif.save()
        pass


## -*- encoding: utf-8 -*-
import os
import sys
from setuptools import setup
from codecs import open # To open the README file with proper encoding
from setuptools.command.test import test as TestCommand # for tests

# Get information from separate files (README, VERSION)
def readfile(filename):
    with open(filename,  encoding='utf-8') as f:
        return f.read()

# For the tests
class SageTest(TestCommand):
    def run_tests(self):
        errno = os.system("sage -t --force-lib carlin")
        if errno != 0:
            sys.exit(1)

setup(
    name = "carlin",
    version = readfile("VERSION"), # the VERSION file is shared with the documentation
    description='Carleman linearization of ordinary differential equations',
    long_description = readfile("README.rst"), # get the long description from the README
    url='https://github.com/mforets/carlin',
    author='Marcelo Forets',
    author_email='marcelo.forets-irurtia@univ-grenoble-alpes.fr', # choose a main contact email
    license='GPLv3', # This should be consistent with the LICENCE file
    classifiers=[
      # How mature is this project? Common values are
      #   3 - Alpha
      #   4 - Beta
      #   5 - Production/Stable
      'Development Status :: 3 - Alpha',
      'Intended Audience :: Science/Research',
      'Topic :: Scientific/Engineering :: Mathematics',
      'License :: OSI Approved :: GNU General Public License v3 (GPLv3)',
      'Programming Language :: Python :: 2.7',
    ], # classifiers list: https://pypi.python.org/pypi?%3Aaction=list_classifiers
    keywords = "sage differential-equations polynomials",
    packages = ['carlin'],
    cmdclass = {'test': SageTest} # adding a special setup command for tests
)