import os
import pandas as pd
import matplotlib.pyplot as plt

# %%

path = r"F:\Загрузки\sms-spam-corpus.csv"
df = pd.read_csv(path, encoding='latin1', names=["v1", "v2", "A", "B", "C"])
df.head(10)

# %%

import re

df['v2'].replace(regex=True, inplace=True, to_replace=r'[^a-zA-Z^ \t\n\r]', value=r'')
df['v2'] = df['v2'].astype(str).map(lambda x: x.lower())

df.head()['v2']

# %%

lineofsw = "''a,to,the,in,have,has,had,do,does,did,am,is,are,shall,will,should,would,may,might,must,can,could,a,to,the,in,be,being,been"

stop_words = lineofsw.split(',')

df['v2'] = df['v2'].apply(lambda x: ' '.join([word for word in x.split(' ') if word not in stop_words]))

df.head()['v2']

# %%

import nltk

ps = nltk.stem.SnowballStemmer('english')
df['v2'] = df['v2'].apply(lambda x: ' '.join([ps.stem(word) for word in x.split(' ')]))

df.to_csv('dataset.csv', index=False)
df.head()['v2']

# %%

from collections import defaultdict

filewithham = open('ham.txt', 'w')
filewithspam = open('spam.txt', 'w')

ham = df[df['v1'] == 'ham']['v2']
spam = df[df['v1'] == 'spam']['v2']

hamdict = defaultdict(int)
spamdict = defaultdict(int)

for sentence in ham:
    for word in sentence.split():
        hamdict[word] += 1

for key, item in hamdict.items():
    filewithham.write('{} {}'.format(key, item) + '\n')

for sentence in spam:
    for word in sentence.split():
        spamdict[word] += 1

for key, item in spamdict.items():
    filewithspam.write('{} {}'.format(key, item) + '\n')

ham_len = hamdict.keys()
spam_len = spamdict.keys()

# %%

import seaborn as sns

fig, ax = plt.subplots()

ham_data = [len(x) for x in ham_len]  # data
filewithham = pd.DataFrame({'ham_len': ham_data})  # dataframe
fig = sns.distplot(filewithham['ham_len']);  # name
fig.figure.savefig('fig_1.png')  # save

# %%

plt.hist(filewithham['ham_len'])
plt.xlabel('ham words')
plt.ylabel('len')
plt.tight_layout()
plt.savefig('fig_2.png')

# %%

spam_data = [len(x) for x in spam_len]
filewithspam = pd.DataFrame({'spam_len': spam_data})

fig = sns.distplot(filewithspam['spam_len']);
fig.figure.savefig('fig_3.png')

# %%

plt.hist(filewithspam['spam_len'])
plt.xlabel('spam words')
plt.ylabel('len')
plt.tight_layout()
plt.savefig('fig_4.png')

# %%

import numpy as np

print('Ham mean: {} | Spam mean: {}'.format(np.mean(ham_data), np.mean(spam_data)))

# %%

ham_msg = [len(x) for x in ham]
filewithham = pd.DataFrame({'ham_msg': ham_msg})
fig = sns.distplot(filewithham['ham_msg']);
fig.figure.savefig('fig_5.png')

# %%

plt.hist(filewithham / filewithham.count())
plt.xlabel('ham msg')
plt.ylabel('len')
plt.tight_layout()
plt.savefig('fig_6.png')

# %%

spam_msg = [len(x) for x in spam]
filewithspam = pd.DataFrame({'spam_msg': spam_msg})
fig = sns.distplot(filewithspam['spam_msg']);
fig.figure.savefig('fig_7.png')

# %%

plt.hist(filewithspam['spam_msg'])
plt.xlabel('spam msg')
plt.ylabel('len')
plt.tight_layout()
plt.savefig('fig_8.png')

# %%

print('Ham msg mean: {} | Spam msg mean: {}'.format(np.mean(ham_msg), np.mean(spam_msg)))

# %%

list_ham = list(hamdict.items())
list_ham.sort(key=lambda i: i[1])
list_ham = list_ham[-20:]
list_ham = dict(list_ham)

list_spam = list(spamdict.items())
list_spam.sort(key=lambda i: i[1])
list_spam = list_spam[-20:]
list_spam = dict(list_spam)

# %%

plt.bar(list_ham.keys(), list_ham.values(), color='black')
plt.xticks(rotation=90);
plt.savefig('fig_9.png')

# %%

plt.bar(list_spam.keys(), list_spam.values(), color='blue')
plt.xticks(rotation=90);
plt.savefig('fig_10.png')

# %%

# %%

# %%

# %%

# %%

# %%


from flask import redirect, render_template, request, abort, session, flash, url_for
from app.db import connection
from app.models.user import User
from app.helpers.auth import authenticated
from app.models.folder import Folder
from app.models.task import Task

def show():
    if not authenticated(session):
        abort(403)
    else:
        id_folder = request.args.get("id_folder")
        if(id_folder == None) or (id_folder == ""):
            flash("The 'id' field is empty!", category="error")
            return redirect(url_for("user-main-menu"))
        conn = connection()
        if User.has_permision_to_open_folder(conn, session['user']['id'], id_folder):
            folder = Folder.find_by_id(conn, id_folder)
            if folder:
                tasks = Task.find_by_id_folder(conn, folder['id'])
                return render_template("folder/folder-show.html", folder=folder, tasks=tasks)
            else:
                flash("The folder doesn't exist!", category="error")
        else:
            flash("You don't have permission to access that folder!", category="error")
        return redirect(url_for("user-main-menu"))

def create():
    if not authenticated(session):
        abort(403)
    else:
        folder_name = request.form.get("folder_name")
        if(folder_name == None) or (folder_name == ""):
            flash("The folder must have a name!", category="error")
            return redirect(url_for("user-main-menu"))
        conn = connection()
        if(Folder.exists(conn, session['user']['id'], folder_name)):
            flash("The folder already exists!", category="error")
        else:
            try:
                Folder.create(conn, session['user']['id'], folder_name)
                flash("The folder was created!", category="success")
            except:
                flash("There was an error", category="error")
        return redirect(url_for("user-main-menu"))

def delete():
    if not authenticated(session):
        abort(403)
    else:
        id_folder = request.form.get("id_folder")
        if(id_folder == None) or (id_folder == ""):
            flash("The folder must have an ID!", category="error")
            return redirect(url_for("user-main-menu"))

        conn = connection()
        folder = Folder.find_by_id(conn, id_folder)
        if not folder:
            flash("The folder doesn't exist!", category="error")
        else:
            if folder['id_user'] == session['user']['id']:
                try:
                    Folder.delete(conn, session['user']['id'], id_folder)
                    flash("The folder was deleted!", category="success")
                except:
                    flash("There was an error!", category="error")
            else:
                flash("You don't have access to that folder!", category="error")
        return redirect(url_for("user-main-menu"))

def main_menu():
    if not authenticated(session):
        abort(403)
    else:
        conn = connection()
        folders = Folder.find_by_id_user(conn, session['user']['id'])
        return render_template("user/menu.html", folders=folders)


#!/usr/bin/env python3

import os
import sys

import requests

import datasets

def main():
    os.makedirs('working', exist_ok=True)
    for name, url in datasets.DATASETS.items():
        print('Downloading {} dataset... '.format(name), end='', flush=True)
        req = requests.get(url)
        if req.status_code != 200:
            print('Error getting {} dataset'.format(name),
                  file=sys.stderr,
                  flush=True)
            continue
        else:
            print('Done', flush=True)
        outpath = datasets.path(name)
        with open(outpath, 'w') as outfile:
            outfile.write(req.text)

if __name__ == '__main__':
    main()


"""
manages a cad project

Alexander "Gilly" Gill
alexgillygill@gmail.com
https://gilly.tk
"""

import sys

if sys.version_info[0] < 3:
    raise Exception('cm needs python 3 or later.')

import argparse
import glob
import os

PARTLIST = 'partlist.csv'
FILELIST = 'filelist.csv'

PARTNUMBER = 'partnumber'
DESCRIPTION = 'description'
PARLOC = 'par file location'
STATUS = 'status'
PATH = 'file'
PARTLISTFORMAT = [PARTNUMBER, DESCRIPTION, PARLOC, STATUS]

FILELISTFORMAT = [PATH, PARTNUMBER]

SOURCEDIR = 'source'
BUILDDIR  = 'build'
EXTENSIONS = ['.par', '.pdf', '.dft', '.dxf', '.stl', '.x_t']

NEWPASTA = 'overwrite this file with a part file'

class Part:
    """a single part"""
    def __init__(self):
        self.partnumber = ''
        self.description = 'uninitialised part'
        self.location = ''
        self.status = 'active'
    def activate(self):
        """set the part to active"""
        self.status = 'active'
    def deactivate(self):
        """set the part to inactive"""
        self.status = 'inactive'

def openpartlist():
    """opens and returns PARTLIST for reading and writing"""
    # TODO: read file after open and initialise if empty
    if os.path.isfile(PARTLIST):
        return open(PARTLIST, 'a+', newline='')
    else:
        # no partlist, so create one
        partlist = open(PARTLIST, 'a+', newline='')
        import csv
        writer = csv.writer(partlist)
        writer.writerow(PARTLISTFORMAT)
        return partlist

def openfilelist():
    """opens and returns FILELIST for reading and writing"""
    # TODO: read file after open and initialise if empty
    if os.path.isfile(FILELIST):
        return open(FILELIST, 'a+', newline='')
    else:
        # no partlist, so create one
        filelist = open(FILELIST, 'a+', newline='')
        import csv
        writer = csv.writer(filelist)
        writer.writerow(FILELISTFORMAT)
        return filelist

def exists(string, csvfile, column):
    """returns whether string is in column of csvfile"""
    # move to begining of file so it can all be read
    csvfile.seek(0, os.SEEK_SET)

    import csv
    reader = csv.reader(csvfile)

    # search entire partlist for partnumber in collumn partnumber_loc
    for entry in reader:
        try:
            if entry[column] == string:
                return True
        except IndexError: # column is not in this row. ignore
            pass

    return False

def get_new_partnumber(partlist):
    """creates a new partnumber unique to partlist"""
    from random import randrange
    col = PARTLISTFORMAT.index(PARTNUMBER)
    # try 60,000 times to get a new partnumber
    for _ in range(0, 60000):
        new_partnumber = format(randrange(0xffff), '04x')
        if not exists(new_partnumber, partlist, col):
            return new_partnumber
    # couldn't find a unique one
    raise Exception("couldn't find a new unique partnumber")
    #TODO: handle this properly

def get_active_partnumbers(partlist):
    # move to begining so all of partlist can be read
    partlist.seek(0, os.SEEK_SET)

    import csv
    reader = csv.reader(partlist)

    # search in STATUS column for 'active'
    status_column = PARTLISTFORMAT.index(STATUS)
    partnumber_column = PARTLISTFORMAT.index(PARTNUMBER)
    for entry in reader:
        try:
            if entry[status_column] == 'active':
                yield entry[partnumber_column]
        except IndexError: # status_column is not in this row. ignore
            pass

def get_file_locations_by_partnumber(partnumber, filelist):
    # move to begining of filelist so it can all be read
    filelist.seek(0, os.SEEK_SET)

    import csv
    reader = csv.reader(filelist)

    # search in PARTNUMBER column for partnumber
    partnumber_column = FILELISTFORMAT.index(PARTNUMBER)
    file_loc_column = FILELISTFORMAT.index(PATH)
    for entry in reader:
        try:
            if entry[partnumber_column] == partnumber:
                yield entry[file_loc_column]
        except IndexError: # one of the columns doesn't exist
            pass

def add_part(part, partlist):
    """adds a part to partlist"""

    # check if the part is already in partlist
    if exists(part.partnumber, partlist, PARTLISTFORMAT.index(PARTNUMBER)):
        print('part is already in partlist: ' + part.partnumber)
    else:
        import csv
        writer = csv.writer(partlist)
        # add to end of partlist
        partlist.seek(0, os.SEEK_END)
        writer.writerow([part.partnumber, part.description, part.location,
                         part.status])

def add_file(path, partnumber, filelist):
    """adds a file to filelist"""
    # check if file is already in FILELIST
    if not exists(path, filelist, FILELISTFORMAT.index(PATH)):
        import csv
        writer = csv.writer(filelist)
        # add to end of filelist
        filelist.seek(0, os.SEEK_END)
        writer.writerow([path, partnumber])
    else: print('file is already in filelist: ' + path)

def parse_args(args):
    """returns the arguments the user gave"""
    parser = argparse.ArgumentParser(description='manage a cad project')
    parser.add_argument('command', choices=['add'],
                        help='the cm command to run')
    parser.add_argument('options', nargs='*',
                        help='options for the command to take')
    args = parser.parse_args(args)
    return args

def add(options):
    """add some files to filelist.csv"""
    parser = argparse.ArgumentParser(description='add a file', prog='cm add')
    parser.add_argument('files', nargs='*', help='the files to add')
    opts = parser.parse_args(options)

    # interate over files in options, and add them to filelist
    with openpartlist() as partlist, openfilelist() as filelist:
        for filereq in opts.files:

            if os.path.isdir(filereq):
                locations = glob.glob(filereq + '/**', recursive=True)
            else:
                locations = glob.glob('**/' + filereq, recursive=True)

            for fileloc in locations:
                # do nothing if the file isn't in SOURCEDIR
                if os.path.normpath(fileloc).split(os.sep)[0] != SOURCEDIR:
                    continue

                # do nothing if the file is not of allowed type
                if os.path.splitext(fileloc)[1] not in EXTENSIONS:
                    continue

                #get partnumber
                print("which part does this file belong to: "
                      + os.path.basename(fileloc))
                partnumber = input('partnumber: ')

                # if partnumber does not exist, make a new part and add it to
                # partlist
                if not exists(partnumber, partlist,
                              PARTLISTFORMAT.index(PARTNUMBER)):
                    part = Part()
                    part.partnumber = partnumber

                    # if this is the par file, add it as location
                    if os.path.splitext(fileloc)[1] == '.par':
                        part.location = fileloc
                    else:
                        part.location = 'err: no par location'

                    # get a description for the part
                    print('give this part a description')
                    part.description = input('eg. "Drive Shaft" > ')

                    add_part(part, partlist)
                #add file to filelist
                add_file(fileloc, partnumber, filelist)

def safe_open_write(path):
    """open a new file when directory may not exist"""
    os.makedirs(os.path.dirname(path), exist_ok=True)
    return open(path, 'x')

def new(options):
    parser = argparse.ArgumentParser(description='make a new managed part',
                                     prog='cm new')
    parser.add_argument('-d', '--description', required=True,
                        help='a description of the new part')
    parser.add_argument('-p', '--partnumber',
                        help='optionally give it a partnumber')
    parser.add_argument('-l', '--location', default=SOURCEDIR,
                        help='optionally define a location for the part')
    opts = parser.parse_args(options)

    partlist = openpartlist()

    part = Part()
    part.description = opts.description
    if opts.partnumber != None:
        part.partnumber = opts.partnumber
    else:
        part.partnumber = get_new_partnumber(partlist)

    # strip punctuation and make a standard filename
    import re
    filename = (re.sub(r'[^\w\s]','',part.description).lower().replace(' ','-')
               + '-' + part.partnumber)
    part.location = os.path.join(opts.location, filename) + EXTENSIONS[0]

    # save a new file to the location
    try:
        with safe_open_write(part.location) as newfile:
            newfile.write(NEWPASTA)
            print('saved a new file to ' + part.location)
            print(NEWPASTA)
            add_part(part, openpartlist())
            add_file(part.location, part.partnumber, openfilelist())
    except FileExistsError:
        print('file {0} already exists in that location'.format(filename))

def build(options):
    """puts all active files into BUILDDIR"""
    from shutil import rmtree, copyfile
    rmtree(BUILDDIR, ignore_errors=True)

    with openpartlist() as partlist, openfilelist() as filelist:
        for partnumber in get_active_partnumbers(partlist):
            for source_loc in get_file_locations_by_partnumber(partnumber, filelist):
                destination = os.path.join(BUILDDIR, partnumber)
                filename = partnumber + os.path.splitext(source_loc)[1]
                os.makedirs(destination, exist_ok=True)
                copyfile(source_loc, os.path.join(destination, filename))

def main():
    """execute when not being loaded as a library"""
    try:
        command = sys.argv[1]
        if command == 'add':
            add(sys.argv[2:])
        elif command == 'build':
            build(sys.argv[2:])
        elif command == 'remove':
            print('not yet implemented')
        elif command == 'new':
            new(sys.argv[2:])
        else:
            raise UserWarning('no such argument')
    except (IndexError, UserWarning):
        print('usage: cm <command> <options>\nread documentation for details\n')
        raise

if __name__ == "__main__":
    main()


from smtplib import SMTPException

from django.core.mail import send_mail
from django.conf import settings
import django.utils.timezone
from django.utils import timezone

from distribution.models import Message, MailingSettings, Log, Client

def send_mailling(mailing):
    now = timezone.localtime(timezone.now())
    if mailing.start_time <= now <= mailing.end_time:
        for message in mailing.messages.all():
            for client in mailing.clients.all():
                try:
                    result = send_mail(
                        subject=message.title,
                        message=message.text,
                        from_email=settings.EMAIL_HOST_USER,
                        recipient_list=[client.email],
                        fail_silently=False
                    )
                    log = Log.objects.create(
                        time=mailing.start_time,
                        status=result,
                        server_response='OK',
                        mailing_list=mailing,
                        client=client
                    )
                    log.save()
                    return log
                except SMTPException as error:
                    log = Log.objects.create(
                        time=mailing.start_time,
                        status=False,
                        server_response=error,
                        mailing_list=mailing,
                        client=client
                    )
                    log.save()
                return log
    else:
        mailing.status = MailingSettings.COMPLETED
        mailing.save()


from copy import deepcopy
from typing import Any, Dict, List, Optional, Tuple

import tensorflow as tf
from tensorflow.keras import Model, Sequential, layers

from doctr.datasets import VOCABS
from doctr.utils.repr import NestedObject

from ...classification import resnet31
from ...utils.tensorflow import _bf16_to_float32, load_pretrained_params
from ..core import RecognitionModel, RecognitionPostProcessor

__all__ = ["SAR", "sar_resnet31"]

default_cfgs: Dict[str, Dict[str, Any]] = {
    "sar_resnet31": {
        "mean": (0.694, 0.695, 0.693),
        "std": (0.299, 0.296, 0.301),
        "input_shape": (32, 128, 3),
        "vocab": VOCABS["french"],
        "url": "https://doctr-static.mindee.com/models?id=v0.6.0/sar_resnet31-c41e32a5.zip&src=0",
    },
}

class SAREncoder(layers.Layer, NestedObject):
    """Implements encoder module of the SAR model

    Args:
        rnn_units: number of hidden rnn units
        dropout_prob: dropout probability
    """

    def __init__(self, rnn_units: int, dropout_prob: float = 0.0) -> None:
        super().__init__()
        self.rnn = Sequential(
            [
                layers.LSTM(units=rnn_units, return_sequences=True, recurrent_dropout=dropout_prob),
                layers.LSTM(units=rnn_units, return_sequences=False, recurrent_dropout=dropout_prob),
            ]
        )

    def call(
        self,
        x: tf.Tensor,
        **kwargs: Any,
    ) -> tf.Tensor:
        # (N, C)
        return self.rnn(x, **kwargs)

class AttentionModule(layers.Layer, NestedObject):
    """Implements attention module of the SAR model

    Args:
        attention_units: number of hidden attention units

    """

    def __init__(self, attention_units: int) -> None:
        super().__init__()
        self.hidden_state_projector = layers.Conv2D(
            attention_units,
            1,
            strides=1,
            use_bias=False,
            padding="same",
            kernel_initializer="he_normal",
        )
        self.features_projector = layers.Conv2D(
            attention_units,
            3,
            strides=1,
            use_bias=True,
            padding="same",
            kernel_initializer="he_normal",
        )
        self.attention_projector = layers.Conv2D(
            1,
            1,
            strides=1,
            use_bias=False,
            padding="same",
            kernel_initializer="he_normal",
        )
        self.flatten = layers.Flatten()

    def call(
        self,
        features: tf.Tensor,
        hidden_state: tf.Tensor,
        **kwargs: Any,
    ) -> tf.Tensor:
        [H, W] = features.get_shape().as_list()[1:3]
        # shape (N, H, W, vgg_units) -> (N, H, W, attention_units)
        features_projection = self.features_projector(features, **kwargs)
        # shape (N, 1, 1, rnn_units) -> (N, 1, 1, attention_units)
        hidden_state = tf.expand_dims(tf.expand_dims(hidden_state, axis=1), axis=1)
        hidden_state_projection = self.hidden_state_projector(hidden_state, **kwargs)
        projection = tf.math.tanh(hidden_state_projection + features_projection)
        # shape (N, H, W, attention_units) -> (N, H, W, 1)
        attention = self.attention_projector(projection, **kwargs)
        # shape (N, H, W, 1) -> (N, H * W)
        attention = self.flatten(attention)
        attention = tf.nn.softmax(attention)
        # shape (N, H * W) -> (N, H, W, 1)
        attention_map = tf.reshape(attention, [-1, H, W, 1])
        glimpse = tf.math.multiply(features, attention_map)
        # shape (N, H * W) -> (N, C)
        return tf.reduce_sum(glimpse, axis=[1, 2])

class SARDecoder(layers.Layer, NestedObject):
    """Implements decoder module of the SAR model

    Args:
        rnn_units: number of hidden units in recurrent cells
        max_length: maximum length of a sequence
        vocab_size: number of classes in the model alphabet
        embedding_units: number of hidden embedding units
        attention_units: number of hidden attention units
        num_decoder_cells: number of LSTMCell layers to stack
        dropout_prob: dropout probability

    """

    def __init__(
        self,
        rnn_units: int,
        max_length: int,
        vocab_size: int,
        embedding_units: int,
        attention_units: int,
        num_decoder_cells: int = 2,
        dropout_prob: float = 0.0,
    ) -> None:
        super().__init__()
        self.vocab_size = vocab_size
        self.max_length = max_length

        self.embed = layers.Dense(embedding_units, use_bias=False)
        self.embed_tgt = layers.Embedding(embedding_units, self.vocab_size + 1)

        self.lstm_cells = layers.StackedRNNCells(
            [layers.LSTMCell(rnn_units, implementation=1) for _ in range(num_decoder_cells)]
        )
        self.attention_module = AttentionModule(attention_units)
        self.output_dense = layers.Dense(self.vocab_size + 1, use_bias=True)
        self.dropout = layers.Dropout(dropout_prob)

    def call(
        self,
        features: tf.Tensor,
        holistic: tf.Tensor,
        gt: Optional[tf.Tensor] = None,
        **kwargs: Any,
    ) -> tf.Tensor:
        if gt is not None:
            gt_embedding = self.embed_tgt(gt, **kwargs)

        logits_list: List[tf.Tensor] = []

        for t in range(self.max_length + 1):  # 32
            if t == 0:
                # step to init the first states of the LSTMCell
                states = self.lstm_cells.get_initial_state(
                    inputs=None, batch_size=features.shape[0], dtype=features.dtype
                )
                prev_symbol = holistic
            elif t == 1:
                # step to init a 'blank' sequence of length vocab_size + 1 filled with zeros
                # (N, vocab_size + 1) --> (N, embedding_units)
                prev_symbol = tf.zeros([features.shape[0], self.vocab_size + 1])
                prev_symbol = self.embed(prev_symbol, **kwargs)
            else:
                if gt is not None:
                    # (N, embedding_units) -2 because of <bos> and <eos> (same)
                    prev_symbol = self.embed(gt_embedding[:, t - 2], **kwargs)
                else:
                    # -1 to start at timestep where prev_symbol was initialized
                    index = tf.argmax(logits_list[t - 1], axis=-1)
                    # update prev_symbol with ones at the index of the previous logit vector
                    # (N, embedding_units)
                    index = tf.ones_like(index)
                    prev_symbol = tf.scatter_nd(
                        tf.expand_dims(index, axis=1),
                        prev_symbol,
                        tf.constant([features.shape[0], features.shape[-1]], dtype=tf.int64),
                    )

            # (N, C), (N, C)  take the last hidden state and cell state from current timestep
            _, states = self.lstm_cells(prev_symbol, states, **kwargs)
            # states = (hidden_state, cell_state)
            hidden_state = states[0][0]
            # (N, H, W, C), (N, C) --> (N, C)
            glimpse = self.attention_module(features, hidden_state, **kwargs)
            # (N, C), (N, C) --> (N, 2 * C)
            logits = tf.concat([hidden_state, glimpse], axis=1)
            logits = self.dropout(logits, **kwargs)
            # (N, vocab_size + 1)
            logits_list.append(self.output_dense(logits, **kwargs))

        # (max_length + 1, N, vocab_size + 1) --> (N, max_length + 1, vocab_size + 1)
        return tf.transpose(tf.stack(logits_list[1:]), (1, 0, 2))

class SAR(Model, RecognitionModel):
    """Implements a SAR architecture as described in `"Show, Attend and Read:A Simple and Strong Baseline for
    Irregular Text Recognition" <https://arxiv.org/pdf/1811.00751.pdf>`_.

    Args:
        feature_extractor: the backbone serving as feature extractor
        vocab: vocabulary used for encoding
        rnn_units: number of hidden units in both encoder and decoder LSTM
        embedding_units: number of embedding units
        attention_units: number of hidden units in attention module
        max_length: maximum word length handled by the model
        num_decoder_cells: number of LSTMCell layers to stack
        dropout_prob: dropout probability for the encoder and decoder
        exportable: onnx exportable returns only logits
        cfg: dictionary containing information about the model
    """

    _children_names: List[str] = ["feat_extractor", "encoder", "decoder", "postprocessor"]

    def __init__(
        self,
        feature_extractor,
        vocab: str,
        rnn_units: int = 512,
        embedding_units: int = 512,
        attention_units: int = 512,
        max_length: int = 30,
        num_decoder_cells: int = 2,
        dropout_prob: float = 0.0,
        exportable: bool = False,
        cfg: Optional[Dict[str, Any]] = None,
    ) -> None:
        super().__init__()
        self.vocab = vocab
        self.exportable = exportable
        self.cfg = cfg
        self.max_length = max_length + 1  # Add 1 timestep for EOS after the longest word

        self.feat_extractor = feature_extractor

        self.encoder = SAREncoder(rnn_units, dropout_prob)
        self.decoder = SARDecoder(
            rnn_units,
            self.max_length,
            len(vocab),
            embedding_units,
            attention_units,
            num_decoder_cells,
            dropout_prob,
        )

        self.postprocessor = SARPostProcessor(vocab=vocab)

    @staticmethod
    def compute_loss(
        model_output: tf.Tensor,
        gt: tf.Tensor,
        seq_len: tf.Tensor,
    ) -> tf.Tensor:
        """Compute categorical cross-entropy loss for the model.
        Sequences are masked after the EOS character.

        Args:
            gt: the encoded tensor with gt labels
            model_output: predicted logits of the model
            seq_len: lengths of each gt word inside the batch

        Returns:
            The loss of the model on the batch
        """
        # Input length : number of timesteps
        input_len = tf.shape(model_output)[1]
        # Add one for additional <eos> token
        seq_len = seq_len + 1
        # One-hot gt labels
        oh_gt = tf.one_hot(gt, depth=model_output.shape[2])
        # Compute loss
        cce = tf.nn.softmax_cross_entropy_with_logits(oh_gt, model_output)
        # Compute mask
        mask_values = tf.zeros_like(cce)
        mask_2d = tf.sequence_mask(seq_len, input_len)
        masked_loss = tf.where(mask_2d, cce, mask_values)
        ce_loss = tf.math.divide(tf.reduce_sum(masked_loss, axis=1), tf.cast(seq_len, model_output.dtype))
        return tf.expand_dims(ce_loss, axis=1)

    def call(
        self,
        x: tf.Tensor,
        target: Optional[List[str]] = None,
        return_model_output: bool = False,
        return_preds: bool = False,
        **kwargs: Any,
    ) -> Dict[str, Any]:
        features = self.feat_extractor(x, **kwargs)
        # vertical max pooling --> (N, C, W)
        pooled_features = tf.reduce_max(features, axis=1)
        # holistic (N, C)
        encoded = self.encoder(pooled_features, **kwargs)

        if target is not None:
            gt, seq_len = self.build_target(target)
            seq_len = tf.cast(seq_len, tf.int32)

        if kwargs.get("training", False) and target is None:
            raise ValueError("Need to provide labels during training for teacher forcing")

        decoded_features = _bf16_to_float32(
            self.decoder(features, encoded, gt=None if target is None else gt, **kwargs)
        )

        out: Dict[str, tf.Tensor] = {}
        if self.exportable:
            out["logits"] = decoded_features
            return out

        if return_model_output:
            out["out_map"] = decoded_features

        if target is None or return_preds:
            # Post-process boxes
            out["preds"] = self.postprocessor(decoded_features)

        if target is not None:
            out["loss"] = self.compute_loss(decoded_features, gt, seq_len)

        return out

class SARPostProcessor(RecognitionPostProcessor):
    """Post processor for SAR architectures

    Args:
        vocab: string containing the ordered sequence of supported characters
    """

    def __call__(
        self,
        logits: tf.Tensor,
    ) -> List[Tuple[str, float]]:
        # compute pred with argmax for attention models
        out_idxs = tf.math.argmax(logits, axis=2)
        # N x L
        probs = tf.gather(tf.nn.softmax(logits, axis=-1), out_idxs, axis=-1, batch_dims=2)
        # Take the minimum confidence of the sequence
        probs = tf.math.reduce_min(probs, axis=1)

        # decode raw output of the model with tf_label_to_idx
        out_idxs = tf.cast(out_idxs, dtype="int32")
        embedding = tf.constant(self._embedding, dtype=tf.string)
        decoded_strings_pred = tf.strings.reduce_join(inputs=tf.nn.embedding_lookup(embedding, out_idxs), axis=-1)
        decoded_strings_pred = tf.strings.split(decoded_strings_pred, "<eos>")
        decoded_strings_pred = tf.sparse.to_dense(decoded_strings_pred.to_sparse(), default_value="not valid")[:, 0]
        word_values = [word.decode() for word in decoded_strings_pred.numpy().tolist()]

        return list(zip(word_values, probs.numpy().clip(0, 1).tolist()))

def _sar(
    arch: str,
    pretrained: bool,
    backbone_fn,
    pretrained_backbone: bool = True,
    input_shape: Optional[Tuple[int, int, int]] = None,
    **kwargs: Any,
) -> SAR:
    pretrained_backbone = pretrained_backbone and not pretrained

    # Patch the config
    _cfg = deepcopy(default_cfgs[arch])
    _cfg["input_shape"] = input_shape or _cfg["input_shape"]
    _cfg["vocab"] = kwargs.get("vocab", _cfg["vocab"])

    # Feature extractor
    feat_extractor = backbone_fn(
        pretrained=pretrained_backbone,
        input_shape=_cfg["input_shape"],
        include_top=False,
    )

    kwargs["vocab"] = _cfg["vocab"]

    # Build the model
    model = SAR(feat_extractor, cfg=_cfg, **kwargs)
    # Load pretrained parameters
    if pretrained:
        load_pretrained_params(model, default_cfgs[arch]["url"])

    return model

def sar_resnet31(pretrained: bool = False, **kwargs: Any) -> SAR:
    """SAR with a resnet-31 feature extractor as described in `"Show, Attend and Read:A Simple and Strong
    Baseline for Irregular Text Recognition" <https://arxiv.org/pdf/1811.00751.pdf>`_.

    >>> import tensorflow as tf
    >>> from doctr.models import sar_resnet31
    >>> model = sar_resnet31(pretrained=False)
    >>> input_tensor = tf.random.uniform(shape=[1, 64, 256, 3], maxval=1, dtype=tf.float32)
    >>> out = model(input_tensor)

    Args:
        pretrained (bool): If True, returns a model pre-trained on our text recognition dataset

    Returns:
        text recognition architecture
    """

    return _sar("sar_resnet31", pretrained, resnet31, **kwargs)


from django.contrib import admin
from django.urls import path,include
from order import views

urlpatterns = [
    # path('admin/', admin.site.urls),
    # path('', include('order.urls')),
    path("",views.home, name='demo'),
    path('sell',views.sell, name='sell'),
    path('register',views.register, name='register'),
    # path('order/booked',views.booked, name='booked'),
    # path("about", views.about, name='about'),
    # path("services", views.service, name="sevices"),
    path("contact", views.contact, name="contact"),
    path('order',views.orderaction, name ='order'),
    path('home1',views.home1, name='home1'),

]


from spinup.utils.run_utils import ExperimentGrid
from spinup import trpo_tf1
import tensorflow as tf

if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('--cpu', type=int, default=-1)
    parser.add_argument('--num_runs', type=int, default=3)
    parser.add_argument('--data_dir', type=str, default='data/TRPO')
    args = parser.parse_args()
    eg = ExperimentGrid(name='TRPO')
    eg.add('env_name', 'CartPole-v1', '', True)
    eg.add('seed', [10*i for i in range(args.num_runs)])
    eg.add('epochs', 100)
    eg.add('steps_per_epoch', 2000)
    eg.add('gamma', 0.99)
    eg.add('delta', [0.001,0.01])
    eg.add('vf_lr', 0.001)
    eg.add('train_v_iters', 40)
    eg.add('damping_coeff', 0.1)
    eg.add('cg_iters', 10)
    eg.add('backtrack_iters', 10)
    eg.add('backtrack_coeff', 0.8)
    eg.add('lam', 0.97)
    eg.add('max_ep_len', 1000)
    eg.add('ac_kwargs:hidden_sizes', (32,32), 'hid')
    eg.add('ac_kwargs:activation', [tf.nn.relu, tf.nn.tanh], '')
    eg.add('algo', ['npg','trpo'])
    eg.run(trpo_tf1, num_cpu=args.cpu)


from django.urls import path
from .views import GetProfile, RegisterView, LoginView, VerifyEmail

urlpatterns = [
    path('register/', RegisterView.as_view()),
    path('login/', LoginView.as_view()),
    path('profile/', GetProfile.as_view()),
    path('verify_email/', VerifyEmail.as_view())

]


import klepto
import os
import pandas as pd
import numpy as np

def chewing_file(num_sections, data, filename, foldername):
    '''
    LIST, DATAFRAMES, & DICTONARIES ONLY.
    When 'swallowing' a pickle causes your notebook to choke, chew it up into batches with klepto.
    batches object into num sections and saves batchs into folders within foldername
    each folder will be named accordingly, "K_"+filename+"_1", "K_"+filename+"_2", ...
    '''
    d = klepto.archives.dir_archive('%s' % foldername, cached=True, serialized=True)
    sections = int(len(data)/num_sections)
    filterByKey = lambda keys: {x: data[x] for x in keys} #for dictionary

    if type(data) == list:
        for num in range(num_sections-1):
            new_name = filename + "_%s" % (num+1)
            d[new_name] = data[(num*sections):((num+1)*sections)]
        d[filename + "_%s" % (num_sections)] = data[(num_sections-1)*sections:]
    elif type(data) == pd.DataFrame:
        for num in range(num_sections-1):
            new_name = filename + "_%s" % (num+1)
            d[new_name] = data.iloc[(num*sections):((num+1)*sections)]
        d[filename + "_%s" % (num_sections)] = data.iloc[(num_sections-1)*sections:]
    elif type(data) == dict:
        for num in range(num_sections-1):
            new_name = filename + "_%s" % (num+1)
            subkeys = sorted(list(data.keys()))[(num*sections):((num+1)*sections)]
            d[new_name] = filterByKey(subkeys)
        subkeys = sorted(list(data.keys()))[(num_sections-1)*sections:]
        d[filename + "_%s" % (num_sections)] = filterByKey(subkeys)
    else:
        d[filename] = data
        print("data is a", type(data))

    d.dump()
    d.clear()

def puking_file(filename, foldername):
    '''
    LIST, DATAFRAMES, & DICTONARIES ONLY.
    Pukes/returns the pieces/batches of the object previously saved.
    Auto-detects number of folders within foldername that CONTAIN the filename, as such...
    "K_"+filename+"_1", "K_"+filename+"_2", ...
    (CAUTION: do not name your files too similarly when you batch save)
    reforms your object with data from the batched folders and returns
    '''
    folder = os.listdir(foldername)
    files = sorted([s for s in folder if s[2:-2] == filename])

    d = klepto.archives.dir_archive('%s' % foldername, cached=True, serialized=True)
    for file in files:
        d.load(file[2:])
#     print(d.keys())

    if type(d[filename+"_1"]) == pd.DataFrame:
        df = []
        for key in sorted(d.keys()):
            df.append(d[str(key)])
        file = pd.concat(df)
    elif type(d[filename+"_1"]) == list:
        file = []
        for key in sorted(d.keys()):
            file.extend(d[str(key)])
    elif type(d[filename+"_1"]) == dict:
        file = {}
        for sub_dict in sorted(d.keys()):
            for key in d[sub_dict].keys():
                file[key] = d[sub_dict][key]
    else:
        file = d[filename]
        print("data is not a DF, list, or dict")

    print('Filename:', filename,
          '\n# of Folders:', len(d.keys()),
          '\nType:', type(file),
          '\nLen:', len(file))

    d.clear()
    return file


import torch
import torch.nn as nn
from torch.nn import init

class graphConv(nn.Module):

	def __init__(self, input_features, output_features, in_dims, num_k, graph_shift):
		super().__init__()
		self.input_features = input_features
		self.output_features = output_features
		self.num_k = num_k
		self.graph_shift = graph_shift
		# self.weight = Parameter(torch.Tensor(num_k, input_features, output_features))
		self.weight = Parameter(torch.Tensor(num_k, in_dims, in_dims)) #Think about this if you wish to change the input and output features
		init.xavier_uniform(self.weight)

	def forward(self, nodes):
		"""
		nodes -- N*D
		graph_shift -- N*N
		"""
		out = torch.Tensor(list(nodes.size())[0], list(nodes.size())[1], self.output_features)
		for ind in range(list(nodes.size())[0]):
			S = torch.eye(list(nodes.size())[1])
			nodes_embed = torch.zeros(list(nodes.size())[1], self.output_features)
			for k in range(self.num_k):
				# nodes_embed += torch.mm(torch.mm(S, nodes[ind]), self.weight[k])
				nodes_embed += torch.mm(self.weight[k], torch.mm(S, nodes[ind]))
				S = torch.mm(S, self.graph_shift)
			out[ind] = nodes_embed
		return out

class CreateLayers(nn.Module):
	def __init__(self,w_init):
		super().__init__()

		self.w_init = w_init

		self.initializations = {
						'uniform': init.uniform_,
						'normal': init.normal_,
						'dirac': init.dirac_,
						'xavier_uniform': init.xavier_uniform_,
						'xavier_normal': init.xavier_normal_,
						'kaiming_uniform': init.kaiming_uniform_,
						'kaiming_normal': init.kaiming_normal_,
						'orthogonal': init.orthogonal_,
						'ones' : init.ones_
		}
		self.activations = nn.ModuleDict([
				['ELU', nn.ELU()],
				['ReLU', nn.ReLU()],
				['Tanh', nn.Tanh()],
				['LogSigmoid', nn.LogSigmoid()],
				['LeakyReLU', nn.LeakyReLU()],
				['SELU', nn.SELU()],
				['CELU', nn.CELU()],
				['GELU', nn.GELU()],
				['Sigmoid', nn.Sigmoid()],
				['Softmax', nn.Softmax()],
				['LogSoftmax', nn.LogSoftmax()]
		])

	def init_weights(self,m):

		if type(m) == nn.Linear:
			self.initializations[self.w_init](m.weight)

	def create_FCNet(self, in_dim, num_layers, h_dim, h_fn, o_dim, o_fn, keep_prob=1.0):
		'''
			GOAL             : Create FC network with different specifications
			in_dims          : number of input units
			num_layers       : number of layers in FCNet
			h_dim  (int)     : number of hidden units
			h_fn             : activation function for hidden layers (default: tf.nn.relu)
			o_dim  (int)     : number of output units
			o_fn             : activation function for output layers (defalut: None)
			w_init           : initialization for weight matrix (defalut: Xavier)
			keep_prob        : keep probabilty [0, 1]  (if None, dropout is not employed)
		'''

		# default active functions (hidden: relu, out: None)
		if h_fn is None:
			h_fn = 'ReLU'
		if o_fn is None:
			o_fn = None

		layers = []
		for layer in range(num_layers):
			if num_layers == 1:
				layers.append(nn.Linear(in_dim,o_dim))  #Discusss
				if o_fn != None:
					layers.append(self.activations[o_fn])
			else:
				if layer == 0:
					layers.append(nn.Linear(in_dim,h_dim))
					layers.append(self.activations[h_fn])
					if not keep_prob is None:
						layers.append(nn.Dropout(keep_prob))
				elif layer > 0 and layer != (num_layers-1): # layer > 0:
					layers.append(nn.Linear(h_dim,h_dim)) #probably wrong
					layers.append(self.activations[h_fn])
					if not keep_prob is None:
						layers.append(nn.Dropout(keep_prob))
				else: # layer == num_layers-1 (the last layer)
					layers.append(nn.Linear(h_dim,o_dim))
					if o_fn != None:
						layers.append(self.activations[o_fn])

		out = nn.Sequential(*layers)

		if self.w_init != None:
			out.apply(self.init_weights)
		return out

	def create_GCNet(self, num_gcn_layers, graph_shift, num_k, gcn_input_features, gcn_hidden_features, gcn_output_features):
		gcn_layers = []
		for layer in range(num_gcn_layers):
			if num_gcn_layers == 1:
				gcn_layers.append(graphConv(gcn_input_features, gcn_output_features, in_dims, num_k, graph_shift))
				gcn_layers.append(self.activations[h_fn]) #Can change to o_fn
			else:
				if layer == 0:
					gcn_layers.append(graphConv(gcn_input_features, gcn_hidden_features, in_dims, num_k, graph_shift))
					gcn_layers.append(self.activations[h_fn])
				elif layer > 0 and layer != (num_gcn_layers-1):
					gcn_layers.append(graphConv(gcn_hidden_features, gcn_hidden_features, in_dims, num_k, graph_shift))
					gcn_layers.append(self.activations[h_fn])
				else:
					gcn_layers.append(graphConv(gcn_hidden_features, gcn_output_features, in_dims, num_K, graph_shift))
					gcn_layers.append(self.activations[h_fn]) #Can change to o_fn

		out = nn.Sequential(*gcn_layers)
		return out


# qlearningAgents.py
# ------------------
# Licensing Information:  You are free to use or extend these projects for
# educational purposes provided that (1) you do not distribute or publish
# solutions, (2) you retain this notice, and (3) you provide clear
# attribution to UC Berkeley, including a link to http://ai.berkeley.edu.
#
# Attribution Information: The Pacman AI projects were developed at UC Berkeley.
# The core projects and autograders were primarily created by John DeNero
# (denero@cs.berkeley.edu) and Dan Klein (klein@cs.berkeley.edu).
# Student side autograding was added by Brad Miller, Nick Hay, and
# Pieter Abbeel (pabbeel@cs.berkeley.edu).

# from _typeshed import Self
from os import stat
from game import *
from learningAgents import ReinforcementAgent
from featureExtractors import *
import numpy as np
from collections import defaultdict

import random,util,math
# from pacman import GameState

class QLearningAgent(ReinforcementAgent):
    """
      Q-Learning Agent

      Functions you should fill in:
        - computeValueFromQValues
        - computeActionFromQValues
        - getQValue
        - getAction
        - update

      Instance variables you have access to
        - self.epsilon (exploration prob)
        - self.alpha (learning rate)
        - self.discount (discount rate)

      Functions you should use
        - self.getLegalActions(state)
          which returns legal actions for a state
    """
    def __init__(self, **args):
        "You can initialize Q-values here..."
        ReinforcementAgent.__init__(self, **args)
        self._default_val = 0.0
        self.Q = defaultdict(lambda:self._default_val)
        self.epsilon = float(args['epsilon'])
        self.alpha = float(args['alpha'])
        self.gamma = float(args['gamma'])

        self._step = 0
        self._num_visits_dict = defaultdict(int)
    def getQRelevantState(state):
        # return tuple(state.data.agentStates)
        data = state.data.deepCopy()
        for a in data.agentStates:
            a.configuration = Configuration(a.configuration.getPosition(), Directions.STOP )
        data.score=0.0
        return data
        # return state
    def getQValue(self, state, action):
        """
          Returns Q(state,action)
          Should return 0.0 if we have never seen a state
          or the Q node value otherwise
        """
        "*** YOUR CODE HERE ***"
        return self.Q[(QLearningAgent.getQRelevantState(state),action)]

    def setQValue(self,state, action, q_value):
        """
          Returns None
          Update new Q value for the state action pair
        """
        "*** YOUR CODE HERE ***"
        relevant_state = QLearningAgent.getQRelevantState(state)
        self.Q[(relevant_state,action)] = q_value
        # update num visits
        # self._num_visits_dict[relevant_state] += 1
    def getNumStates(self):
        """
        Returns number of visits in state
        """
        return len(list(self.Q.values()))
    def getNumVisits(self, state):
        """
        Returns number of visits in state
        """
        relevant_state = QLearningAgent.getQRelevantState(state)
        return self._num_visits_dict[relevant_state]

    def getNumStatesVisitedLessThan(self,times: int):
        """
        Returns number of states thaat was visited less than `times` argument
        """
        return len([v for v in self._num_visits_dict.values() if v < times])

    def computeValueFromQValues(self, state):
        """
          Returns max_action Q(state,action)
          where the max is over legal actions.  Note that if
          there are no legal actions, which is the case at the
          terminal state, you should return a value of 0.0.
        """
        "*** YOUR CODE HERE ***"
        actions = self.getLegalActions(state)
        # check if terminal state
        if not actions:
          return self._default_val
        # list all q values for current state's actions - this implicitly initilizes unseen state-actions
        list_of_state_action_vals = [self.getQValue(state,a) for a in actions]
        return self._default_val if not list_of_state_action_vals else max(list_of_state_action_vals)

    def computeActionFromQValues(self, state):
        """
          Compute the best action to take in a state.  Note that if there
          are no legal actions, which is the case at the terminal state,
          you should return None.
        """
        "*** YOUR CODE HERE ***"
        actions = self.getLegalActions(state)
        # check if terminal state
        if not actions:
          return None
        best_val = self.computeValueFromQValues(state)
        list_of_best_actions = [a for a in actions if self.getQValue(state,a) == best_val]
        assert list_of_best_actions
        assert all([a in actions for a in list_of_best_actions])
        return random.choice(list_of_best_actions)

    def getAction(self, state):
        """
          Compute the action to take in the current state.  With
          probability self.epsilon, we should take a random action and
          take the best policy action otherwise.  Note that if there are
          no legal actions, which is the case at the terminal state, you
          should choose None as the action.

          HINT: You might want to use util.flipCoin(prob)
          HINT: To pick randomly from a list, use random.choice(list)
        """
        # Pick Action
        "*** YOUR CODE HERE ***"
        self._step += 1

        actions = self.getLegalActions(state)
        # check if terminal state
        if not actions:
          print('max Q value', max(list(self.Q.values())))
          return None
        # epsilon greedy exploration
        if util.flipCoin(self.epsilon):
          return random.choice(actions)

        action_from_Q_values = self.computeActionFromQValues(state)

        chosen_action = action_from_Q_values if action_from_Q_values is not None else random.choice(actions)

        # update num visits
        relevant_state = QLearningAgent.getQRelevantState(state)
        self._num_visits_dict[relevant_state] += 1

        return chosen_action

    def update(self, state, action, nextState, reward):
        """
          The parent class calls this to observe a
          state = action => nextState and reward transition.
          You should do your Q-Value update here

          NOTE: You should never call this function,
          it will be called on your behalf
        """
        "*** YOUR CODE HERE ***"
        next_state_max_val = self.computeValueFromQValues(nextState)
        s = state.data.agentStates
        # self.Q[(state,action)] = self.Q[(state,action)] + self.alpha * (reward +  self.gamma*next_state_max_val - self.Q[(state,action)])
        Q_sa = self.getQValue(state,action)
        Q_sa += self.alpha * (reward +  self.gamma*next_state_max_val - Q_sa)
        Q_sa = float(int(Q_sa * 100))/100.0
        self.setQValue(state,action,Q_sa)

    def getPolicy(self, state):
        return self.computeActionFromQValues(state)

    def getValue(self, state):
        return self.computeValueFromQValues(state)

class PacmanQAgent(QLearningAgent):
    "Exactly the same as QLearningAgent, but with different default parameters"

    def __init__(self, epsilon=0.05,gamma=0.8,alpha=0.2, numTraining=0, **args):
        """
        These default parameters can be changed from the pacman.py command line.
        For example, to change the exploration rate, try:
            python pacman.py -p PacmanQLearningAgent -a epsilon=0.1
        alpha    - learning rate
        epsilon  - exploration rate
        gamma    - discount factor
        numTraining - number of training episodes, i.e. no learning after these many episodes
        """
        args['epsilon'] = epsilon
        args['gamma'] = gamma
        args['alpha'] = alpha
        args['numTraining'] = numTraining
        self.index = 0  # This is always Pacman
        QLearningAgent.__init__(self, **args)

    def getAction(self, state):
        """
        Simply calls the getAction method of QLearningAgent and then
        informs parent of action for Pacman.  Do not change or remove this
        method.
        """
        action = QLearningAgent.getAction(self, state)
        self.doAction(state, action)
        return action


import pygame
import random
from pygame.constants import *

pygame.init()
screen = pygame.display.set_mode((800, 600))
screen.fill((255, 255, 255))

surf = pygame.surface.Surface((800, 250))
surf.fill((229, 198, 135))

blackboard = pygame.surface.Surface((760, 350))
blackboard.fill((239, 224, 224))

screen.blit(surf, (0, 0))
screen.blit(blackboard, (40, 0))
pygame.display.set_caption("Paint")

colors = {
        'red': (255, 0, 0),
        'green': (0, 255, 0),
        'blue': (0, 0, 255),
        'purple': (83, 28, 232),
        'black': (0, 0, 0),
        'random': (random.randint(0, 255), random.randint(0, 255), random.randint(0, 255))
    }

def line(screen, start, end, d, color):
    x1 = start[0]
    y1 = start[1]
    x2 = end[0]
    y2 = end[1]

    dx = abs(x1 - x2)
    dy = abs(y1 - y2)

    A = y2 - y1
    B = x1 - x2
    C = x2 * y1 - x1 * y2

    if dx > dy:
        if x1 > x2:
            x1, x2 = x2, x1
            y1, y2 = y2, y1

        for x in range(x1, x2):
            y = (-C - A * x) / B
            pygame.draw.circle(screen, color, (x, y), d)
    else:
        if y1 > y2:
            x1, x2 = x2, x1
            y1, y2 = y2, y1
        for y in range(y1, y2):
            x = (-C - B * y) / A
            pygame.draw.circle(screen, color, (x, y), d)

def rectangle(screen, start, end, d, color):
    x1 = start[0]
    y1 = start[1]
    x2 = end[0]
    y2 = end[1]

    width = abs(x1 - x2)
    height = abs(y1 - y2)

    if x1 < x2 and y1 < y2:
        pygame.draw.rect(screen, color, [x1, y1, width, height])
    elif x1 < x2 and y2 < y1:
        pygame.draw.rect(screen, color, [x1, y2, width, height])
    elif x2 < x1 and y1 < y2:
        pygame.draw.rect(screen, color, [x2, y1, width, height])
    elif x2 < x1 and y2 < y1:
        pygame.draw.rect(screen, color, [x2, y2, width, height])

def circle(screen, start, end, d, color):
    x1 = start[0]
    y1 = start[1]
    x2 = end[0]
    y2 = end[1]

    width = abs(x1 - x2)
    height = abs(y1 - y2)

    if x1 < x2 and y1 < y2:
        pygame.draw.ellipse(screen, color, [x1, y1, width, height], d)
    elif x1 < x2 and y2 < y1:
        pygame.draw.ellipse(screen, color, [x1, y2, width, height], d)
    elif x2 < x1 and y1 < y2:
        pygame.draw.ellipse(screen, color, [x2, y1, width, height], d)
    elif x2 < x1 and y2 < y1:
        pygame.draw.ellipse(screen, color, [x2, y2, width, height], d)

def info_text(surf):
    pygame.draw.aaline(surf, colors['black'], (40, 0), (40, 250), 2)
    font = pygame.font.SysFont('Times New Roman', 20, False, False)

    t1 = font.render("Alt + R -- rectangle", True, colors['black'], (229, 198, 135))
    t2 = font.render("Alt + L -- line", True, colors['black'], (229, 198, 135))
    t3 = font.render("Alt + C -- circle", True, colors['black'], (229, 198, 135))
    t4 = font.render("Alt + E -- eraser", True, colors['black'], (229, 198, 135))
    t5 = font.render("Alt + Up -- size-up for eraser", True, colors['black'], (229, 198, 135))
    t6 = font.render("Alt + Down -- size-down for eraser", True, colors['black'], (229, 198, 135))

    t7 = font.render("Ctrl + R -- red", True, colors['black'], (229, 198, 135))
    t8 = font.render("Ctrl + G -- green", True, colors['black'], (229, 198, 135))
    t9 = font.render("Ctrl + B -- blue", True, colors['black'], (229, 198, 135))
    t10 = font.render("Ctrl + P -- purple", True, colors['black'], (229, 198, 135))
    t11 = font.render("Ctrl + Space -- random-color", True, colors['black'], (229, 198, 135))
    t12 = font.render("Up -- size-up", True, colors['black'], (229, 198, 135))
    t13 = font.render("Down -- size-down", True, colors['black'], (229, 198, 135))
    t14 = font.render("Ctrl + S -- for save", True, colors['black'], (229, 198, 135))

    surf.blit(t1, (150, 15))
    surf.blit(t2, (150, 40))
    surf.blit(t3, (150, 65))
    surf.blit(t4, (150, 90))
    surf.blit(t5, (150, 115))
    surf.blit(t6, (150, 140))

    surf.blit(t7, (500, 15))
    surf.blit(t8, (500, 40))
    surf.blit(t9, (500, 65))
    surf.blit(t10, (500, 90))
    surf.blit(t11, (500, 115))

    surf.blit(t12, (500, 150))
    surf.blit(t13, (500, 175))
    surf.blit(t14, (500, 200))

done = False
draw_line = False
erase = False

commands = {
    "lining": True,
    "recting": False,
    "circling": False,
    "erasing": False
}
surf_elements = {
    'line': (255, 0, 0),
    'rec': (10, 10, 10),
    'circ': (10, 10, 10),
    'erase': (10, 10, 10)
}
x, y = 0, 0
last_pos = (0, 0)
d = 2
ed = 20
color = (0, 0, 0)
Pi = 3.14

while not done:
    get_pressed = pygame.key.get_pressed()
    alt_press = get_pressed[K_LALT] or get_pressed[K_RALT]
    ctrl_press = get_pressed[K_LCTRL] or get_pressed[K_RCTRL]
    pos = pygame.mouse.get_pos()

    font = pygame.font.SysFont('Verdana', 17, False, False)

    pygame.draw.circle(surf, color, (20, 30), 15)
    pygame.draw.rect(surf, surf_elements['rec'], [5, 60, 30, 30], 1)
    pygame.draw.circle(surf, surf_elements['circ'], [20, 120], 15, 1)
    pygame.draw.arc(surf, surf_elements['line'], [6, 151, 15, 15], Pi, 2 * Pi, 1)
    pygame.draw.arc(surf, surf_elements['line'], [20, 154, 15, 15], 0, Pi, 1)
    pygame.draw.aalines(surf, surf_elements['erase'], True, [[10, 190], [15, 205], [25, 205], [30, 190], [20, 180]])
    size_text = font.render(str(d), True, colors['black'], (229, 198, 135))

    for event in pygame.event.get():
        if event.type == QUIT:
            done = True
        if event.type == KEYDOWN:
            if alt_press and event.key == K_r:
                for i in surf_elements:
                    if i == 'rec':
                        surf_elements[i] = colors['red']
                    else:
                        surf_elements[i] = (10, 10, 10)
                for i in commands:
                    if i == 'recting':
                        commands[i] = True
                    else:
                        commands[i] = False
            if alt_press and event.key == K_l:
                for i in surf_elements:
                    if i == 'line':
                        surf_elements[i] = colors['red']
                    else:
                        surf_elements[i] = (10, 10, 10)
                for i in commands:
                    if i == 'lining':
                        commands[i] = True
                    else:
                        commands[i] = False
            if alt_press and event.key == K_c:
                for i in surf_elements:
                    if i == 'circ':
                        surf_elements[i] = colors['red']
                    else:
                        surf_elements[i] = (10, 10, 10)
                for i in commands:
                    if i == 'circling':
                        commands[i] = True
                    else:
                        commands[i] = False
            if alt_press and event.key == K_e:
                for i in surf_elements:
                    if i == 'erase':
                        surf_elements[i] = colors['red']
                    else:
                        surf_elements[i] = (10, 10, 10)
                for i in commands:
                    if i == 'erasing':
                        commands[i] = True
                    else:
                        commands[i] = False
            if event.key == K_UP:
                if d < 9:
                    d += 1
            if event.key == K_DOWN:
                if d > 1:
                    d -= 1
            if alt_press and event.key == K_UP:
                if ed < 50:
                    ed += 5
            if alt_press and event.key == K_DOWN:
                if ed > 20:
                    ed -= 5
            if ctrl_press and event.key == K_s:
                pygame.image.save(blackboard, 'screenshot.jpg')
            if alt_press and event.key == K_SPACE:
                pygame.image.save(surf, 'screenshot.jpg')
            if ctrl_press and event.key == K_r:
                color = colors['red']
            if ctrl_press and event.key == K_g:
                color = colors['green']
            if ctrl_press and event.key == K_b:
                color = colors['blue']
            if ctrl_press and event.key == K_p:
                color = colors['purple']
            if ctrl_press and event.key == K_SPACE:
                colors['random'] = (random.randint(0, 255), random.randint(0, 255), random.randint(0, 255))
                color = colors['random']

        if commands['recting']:
            if event.type == MOUSEBUTTONDOWN:
                last_pos = pos
            if event.type == MOUSEBUTTONUP:
                rectangle(blackboard, last_pos, pos, d, color)

        if commands['lining']:
            if event.type == MOUSEBUTTONDOWN:
                last_pos = pos
                pygame.draw.circle(blackboard, color, pos, d)
                draw_line = True
            if event.type == MOUSEBUTTONUP:
                draw_line = False
            if event.type == MOUSEMOTION:
                if draw_line:
                    line(blackboard, last_pos, pos, d, color)
                last_pos = pos

        if commands['circling']:
            if event.type == MOUSEBUTTONDOWN:
                last_pos = pos
            if event.type == MOUSEBUTTONUP:
                circle(blackboard, last_pos, pos, d, color)

        if commands['erasing']:
            if event.type == pygame.MOUSEBUTTONDOWN:
                (x, y) = pos
                pygame.draw.rect(blackboard, (239, 224, 224), [x, y, ed, ed])
                erase = True
            if event.type == pygame.MOUSEMOTION:
                if erase:
                    pygame.draw.rect(blackboard, (239, 224, 224), [pos[0], pos[1], ed, ed])
            if event.type == pygame.MOUSEBUTTONUP:
                erase = False

    info_text(surf)
    surf.blit(size_text, (15, 215))
    screen.blit(surf, (0, 350))
    screen.blit(blackboard, (0, 0))
    pygame.display.flip()

pygame.quit()


import mimetypes
import os
from re import X
from unicodedata import name
from flask import Blueprint, appcontext_popped, redirect, render_template, request, flash, jsonify, url_for
from flask_login import login_required, current_user
from werkzeug.utils import secure_filename
from .models import FDP, Note,Events,Workshops,publications,CP,JOURNAL
from . import db
import json
views = Blueprint('views', __name__)

@views.route('/', methods=['GET', 'POST'])
@login_required
def home():
    if request.method == 'POST':
        title = request.form['title']
        author = request.form['author']
        teacher = request.form['teacher']
        journal = request.form['journal']
        year = request.form['year']
        link = request.form['link']
        index = request.form['index']

        new_note = Note(title=title, author=author, department_teacher=teacher,journal=journal,publication_year=year,link=link,h_index=index, user_id=current_user.id)

        db.session.add(new_note)
        db.session.commit()
        flash('added!', category='success')

    return render_template("home.html", user=current_user)

@views.route('/Workshops', methods=['GET', 'POST'])
@login_required
def workshops():
    if request.method == 'POST':
        Academic_Year = request.form['Academic_Year']
        Workshop_Name = request.form['Workshop_Name']
        Proposal_Date = request.form['Proposal_Date']
        Conductin_Date = request.form['Conductin_Date']
        Instituion_Approval = request.form['Instituion_Approval']
        Report = request.form['Report']

        new_Workshops = Workshops(Academic_Year=Academic_Year, Workshop_Name=Workshop_Name, Proposal_Date=Proposal_Date,Conductin_Date=Conductin_Date,Instituion_Approval=Instituion_Approval,Report=Report, user_id=current_user.id)

        db.session.add(new_Workshops)
        db.session.commit()
        flash('Workshops added!', category='success')
    return render_template("workshops.html",user=current_user)

@views.route('/Events', methods=['GET', 'POST'])
@login_required
def events():
    if request.method == 'POST':
        Event_Name = request.form['Event_Name']
        Proposal_Date = request.form['Proposal_Date']
        Conductin_Date = request.form['Conductin_Date']
        Instituion_Approval = request.form['Instituion_Approval']
        Report = request.form['Report']

        new_events = Events(Event_Name=Event_Name, Proposal_Date=Proposal_Date,Conductin_Date=Conductin_Date,Instituion_Approval=Instituion_Approval,Report=Report, user_id=current_user.id)

        db.session.add(new_events)
        db.session.commit()
        flash('Events added!', category='success')
    return render_template("events.html",user=current_user)

@views.route('/Publications', methods=['GET', 'POST'])
@login_required
def publication():
    if request.method == 'POST':
        JournalName_Conference = request.form['JournalName_Conference']
        ISSN_ISBN_Number_Progress = request.form['ISSN_ISBN_Number_Progress']
        Month_and_Year = request.form['Month_and_Year']
        Title = request.form['Title']
        Link = request.form['Link']

        new_publications = publications(JournalName_Conference=JournalName_Conference, ISSN_ISBN_Number_Progress=ISSN_ISBN_Number_Progress,Month_and_Year=Month_and_Year,Title=Title,Link=Link, user_id=current_user.id)

        db.session.add(new_publications)
        db.session.commit()
        flash('publications added!', category='success')
    return render_template("publications.html",user=current_user)

@views.route('/FDP', methods=['GET', 'POST'])
@login_required
def fdp():
    if request.method == 'POST':
        Proposal_Name = request.form['Proposal_Name']
        Organization_Name = request.form['Organization_Name']
        Start_Date = request.form['Start_Date']
        Completion_Date = request.form['Completion_Date']
        Status_Preparation = request.form['Status_Preparation']
        Proposal_Copy = request.form['Proposal_Copy']
        Final_Report = request.form['Final_Report']

        new_FDP = FDP(Proposal_Name=Proposal_Name, Organization_Name=Organization_Name, Start_Date=Start_Date,Completion_Date=Completion_Date,Status_Preparation=Status_Preparation,Proposal_Copy=Proposal_Copy,Final_Report=Final_Report, user_id=current_user.id)

        db.session.add(new_FDP)
        db.session.commit()
        flash('FDP added!', category='success')
    return render_template("FDP.html", user=current_user)

@views.route('/CP', methods=['GET', 'POST'])
@login_required
def cp():
    if request.method == 'POST':
        Proposal_Name = request.form['Proposal_Name']
        Organization_Name = request.form['Organization_Name']
        Start_Date = request.form['Start_Date']
        Completion_Date = request.form['Completion_Date']
        Status_Preparation = request.form['Status_Preparation']
        Proposal_Copy = request.form['Proposal_Copy']
        Final_Report = request.form['Final_Report']

        new_FDP = CP(Proposal_Name=Proposal_Name, Organization_Name=Organization_Name, Start_Date=Start_Date,Completion_Date=Completion_Date,Status_Preparation=Status_Preparation,Proposal_Copy=Proposal_Copy,Final_Report=Final_Report, user_id=current_user.id)

        db.session.add(new_FDP)
        db.session.commit()
        flash('FDP added!', category='success')
    return render_template("cp.html", user=current_user)

@views.route('/JOURNAL', methods=['GET', 'POST'])
@login_required
def journal():
    if request.method == 'POST':
        JournalName = request.form['JournalName']
        ISSN_Number = request.form['ISSN_Number']
        month_and_years = request.form['month_and_years']
        Title = request.form['Title']
        Paper_Link = request.form['Paper_Link']

        new_FDP = JOURNAL(JournalName=JournalName, ISSN_Number=ISSN_Number, month_and_years=month_and_years,Title=Title,Paper_Link=Paper_Link,user_id=current_user.id)

        db.session.add(new_FDP)
        db.session.commit()
        flash('FDP added!', category='success')
    return render_template("journal.html", user=current_user)

@views.route('/download')
@login_required
def download():

    all_data= Note.query.filter_by(user_id=current_user.id).all()

    with open(f'website/static/data/{current_user.id}.csv',"w+") as f:
        for data in all_data:
            row = f"{data.title}, {data.author},{data.department_teacher},{data.journal},{data.publication_year},{data.link},{data.h_index}"
            f.write(row)
            f.write("\n")

    return render_template("download.html", link=f'/static/data/{current_user.id}.csv')

@views.route('/delete-note', methods=['POST'])
def delete_note():
    note = json.loads(request.data)
    noteId = note['noteId']
    note = Note.query.get(noteId)
    if note:
        if note.user_id == current_user.id:
            db.session.delete(note)
            db.session.commit()
        return jsonify({})

@views.route('/delete-events', methods=['POST'])
def delete_Events():
    Events = json.loads(request.data)
    EventsId = Events['EventsId']
    Events = Events.query.get(EventsId)
    if Events:
        if events.user_id == current_user.id:
            db.session.delete(Events)
            db.session.commit()
        return jsonify({})


import json
from pathlib import Path

import requests
from loguru import logger
from picsellia import Client

import config

if __name__ == "__main__":
    # Connect to Picsellia.
    client = Client(
    api_token=config.API_TOKEN,
    organization_name=config.ORGANIZATION_NAME
    )

    # Get deployment by name.
    my_deployment = client.get_deployment(name=config.DEPLOYMENT_NAME)

    # Get your deployment ID.
    deployment_id = str(my_deployment.id)

    # Get your API token from your profile.
    api_token = str(config.API_TOKEN)

    # Authentication url
    auth_url = "https://serving.picsellia.com/api/login"

    headers = {
        "Authorization": "Token " + api_token,
    }

    jwt_generation_data = {
        "deployment_id": deployment_id,
        "api_token": api_token
    }

    jwt_request = requests.post(
        auth_url,
        headers=headers,
        data=json.dumps(jwt_generation_data)
    )

    # Retrieving the JWT.
    jwt = jwt_request.json()["jwt"]

    # Serving API endpoint
    url = "https://serving.picsellia.com/api/deployment/{}/predict".format(my_deployment.id)

    header = {
        "Authorization": "Bearer " + jwt,
    }

    # Metadata that helps extrat data from datalake.
    data = {
        "source": "camera1",
        "tag": "Cartons"
    }

    images_folder = config.INFERENCE_DATA_DIR

    images_path = Path(images_folder).glob('*.jpg')

    for image_path in images_path:
        with open(image_path, "rb") as file:
            r = requests.post(
                url=url,
                files={'media': file},
                headers=header,
                data=data
            )

    logger.info(r.text)

    prediction = r.json()


from datetime import datetime
import logging

logger = logging.getLogger(__name__)

class TimePeriod:
    def __init__(self, datetimes: list):
        self._assess_period_length(datetimes)
        self.period_length = self._define_period_length()

    def _assess_period_length(self, datetime_strings: list):
        timestamps = []

        for datetime_string in datetime_strings:
            try:
                datetime_object = datetime.strptime(
                    datetime_string, "%Y-%m-%d %H:%M:%S")
            except ValueError as e:
                datetime_object = datetime.strptime(
                    datetime_string, "%Y-%m-%dT%H:%M:%SZ")

            timestamp = datetime.timestamp(datetime_object)

            if timestamp > 0:
                timestamps.append(timestamp)

        timestamps.sort()

        self.time_delta = datetime.fromtimestamp(timestamps[-1]) \
                - datetime.fromtimestamp(timestamps[0])
        self.first_datetime = datetime.fromtimestamp(timestamps[0])
        self.last_datetime = datetime.fromtimestamp(timestamps[-1])

    def _define_period_length(self) -> str:
        days_delta = self.time_delta.days
        period_string = ""

        if days_delta ==1:
            period_string = "a day"
        elif days_delta == 2 and self._is_weekend():
            period_string = "a weekend"
        elif days_delta == 2 and not self._is_weekend():
            period_string = "two days"
        elif 3 <= days_delta <= 6:
            period_string = "a few days"
        elif 7 <= days_delta <= 10:
            period_string = "a week"
        elif 11 <= days_delta <= 19:
            period_string = "two weeks"
        elif 20 <= days_delta <= 30:
            period_string = self.get_period_month()
        elif 80  <= days_delta <= 90:
            period_string = "a quarter"
        elif 150  <= days_delta <= 190:
            period_string = "a semester"
        elif 300  <= days_delta <= 400:
            period_string = "a year"

        return period_string

    def _is_weekend(self) -> bool:
        is_weekend = False
        days_delta = self.time_delta.days
        first_day = selffirst_datetime.strftime('%A')
        second_day = self.last_datetime.strftime('%A')

        if days_delta == 2 and \
                (first_day == "Saturday" and second_day == "Sunday"):
            is_weekend = True

        return is_weekend

    def get_period_month(self) -> str:
        months = []

        if slef.period_length == "month":
            month = self.first_datetime.strftime("%B")

        return month


from pm4py import util as pmutil
from pm4py.evaluation.replay_fitness.versions import alignment_based, token_replay
from pm4py.objects.conversion.log import factory as log_conversion
from pm4py.objects.log.util import general as log_util
from pm4py.objects.log.util import xes as xes_util

ALIGNMENT_BASED = "alignments"
TOKEN_BASED = "token_replay"
VERSIONS = {ALIGNMENT_BASED: alignment_based.apply, TOKEN_BASED: token_replay.apply}
VERSIONS_EVALUATION = {ALIGNMENT_BASED: alignment_based.evaluate, TOKEN_BASED: token_replay.evaluate}

PARAM_ACTIVITY_KEY = 'activity_key'

def apply(log, petri_net, initial_marking, final_marking, parameters=None, variant="token_replay"):
    """
    Apply fitness evaluation starting from an event log and a marked Petri net,
    by using one of the replay techniques provided by PM4Py

    Parameters
    -----------
    log
        Trace log object
    petri_net
        Petri net
    initial_marking
        Initial marking
    final_marking
        Final marking
    parameters
        Parameters related to the replay algorithm
    variant
        Chosen variant (alignments or token-based replay)

    Returns
    ----------
    fitness_eval
        Fitness evaluation
    """
    if parameters is None:
        parameters = {}
    if pmutil.constants.PARAMETER_CONSTANT_ACTIVITY_KEY not in parameters:
        parameters[pmutil.constants.PARAMETER_CONSTANT_ACTIVITY_KEY] = xes_util.DEFAULT_NAME_KEY
    if pmutil.constants.PARAMETER_CONSTANT_TIMESTAMP_KEY not in parameters:
        parameters[pmutil.constants.PARAMETER_CONSTANT_TIMESTAMP_KEY] = xes_util.DEFAULT_TIMESTAMP_KEY
    if pmutil.constants.PARAMETER_CONSTANT_CASEID_KEY not in parameters:
        parameters[pmutil.constants.PARAMETER_CONSTANT_CASEID_KEY] = log_util.CASE_ATTRIBUTE_GLUE

    return VERSIONS[variant](log_conversion.apply(log, parameters, log_conversion.TO_EVENT_LOG), petri_net,
                             initial_marking, final_marking, parameters=parameters)

def evaluate(results, parameters="None", variant="token_replay"):
    """
    Evaluate replay results when the replay algorithm has already been applied

    Parameters
    -----------
    results
        Results of the replay algorithm
    parameters
        Possible parameters passed to the evaluation
    variant
        Indicates which evaluator is called

    Returns
    -----------
    fitness_eval
        Fitness evaluation
    """
    return VERSIONS_EVALUATION[variant](results, parameters=parameters)


from django.conf.urls import patterns, url

urlpatterns = patterns(
    "cvmo.cluster.views",
    # Views
    url(r"^new$", "show_new", name="cluster_new"),
    url(r"^clone/(?P<cluster_id>[0-9]+)$", "show_new", name="cluster_new"),
    url(r"^test$", "show_test"),
    # url(r"^edit/(?P<cluster_id>[0-9]+)$", "show_edit", name="cluster_edit"),
    url(r"^deploy/(?P<cluster_id>[0-9]+)$", "show_deploy", name="cluster_deploy"),

    # Actions
    url(r"^save$", "save", name="cluster_save"),
    url(r"^delete/(?P<cluster_id>[0-9]+)$", "delete", name="cluster_delete"),
)


import datetime

import pytest

from sqlalchemy.exc import NoResultFound

from decisionengine.framework.dataspace import dataspace as ds
from decisionengine.framework.dataspace.datablock import Header, Metadata
from decisionengine.framework.dataspace.tests.fixtures import (  # noqa: F401
    DATABASES_TO_TEST,
    dataspace,
    PG_DE_DB_WITHOUT_SCHEMA,
    PG_PROG,
    SQLALCHEMY_PG_WITH_SCHEMA,
    SQLALCHEMY_TEMPFILE_SQLITE,
)

def test_has_config(dataspace):  # noqa: F811
    """verify our config entry exists"""
    assert isinstance(dataspace.config, dict)

def test_dataspace_config_finds_bad():
    with pytest.raises(ds.DataSpaceConfigurationError) as e:
        ds.DataSpace({})
    assert e.match("missing dataspace information")

    with pytest.raises(ds.DataSpaceConfigurationError) as e:
        ds.DataSpace({"dataspace": "asdf"})
    assert e.match("dataspace key must correspond to a dictionary")

    with pytest.raises(ds.DataSpaceConfigurationError) as e:
        ds.DataSpace({"dataspace": {"asdf": "asdf"}})
    assert e.match("Invalid dataspace configuration")

def test_get_taskmanager_exists(dataspace):  # noqa: F811
    """Can I get a taskmanager by name or name and uuid"""
    # should return the 'newest' instance
    result1 = dataspace.get_taskmanager(taskmanager_name="taskmanager1")
    assert result1["name"] == "taskmanager1"
    assert str(result1["taskmanager_id"]) == "11111111-1111-1111-1111-111111111111"

    result2 = dataspace.get_taskmanager(
        taskmanager_name="taskmanager1",
        taskmanager_id="11111111-1111-1111-1111-111111111111",
    )
    assert result2["name"] == "taskmanager1"
    assert str(result2["taskmanager_id"]) == "11111111-1111-1111-1111-111111111111"

    assert result1 == result2

def test_delete(dataspace):  # noqa: F811
    # this doesn't do much at this level, but we can make sure it exists
    dataspace.delete("11111111-1111-1111-1111-111111111111")
    dataspace.delete("22222222-2222-2222-2222-222222222222", all_generations=True)

def test_mark_expired(dataspace):  # noqa: F811
    # this doesn't do much at this level, but we can make sure it exists
    dataspace.mark_expired("11111111-1111-1111-1111-111111111111", 1, "my_test_key", 0)

def test_get_taskmanager_not_exists(dataspace):  # noqa: F811
    """This should error out"""
    with pytest.raises((KeyError, NoResultFound)):
        dataspace.get_taskmanager(taskmanager_name="no_such_task_manager")
    with pytest.raises((KeyError, NoResultFound)):
        dataspace.get_taskmanager(
            taskmanager_name="no_such_task_manager",
            taskmanager_id="11111111-1111-1111-1111-111111111111",
        )

def test_get_taskmanagers(dataspace):  # noqa: F811
    """Can I get multimple task managers"""
    yesterday = str(datetime.datetime.now() - datetime.timedelta(days=1))
    two_years_future = str(datetime.datetime.now() + datetime.timedelta(days=730))

    result0 = dataspace.get_taskmanagers()
    assert len(result0) == 2

    result1 = dataspace.get_taskmanagers(taskmanager_name="taskmanager1")
    assert len(result1) == 1
    assert result1[0]["name"] == "taskmanager1"
    assert str(result1[0]["taskmanager_id"]) == "11111111-1111-1111-1111-111111111111"

    result2 = dataspace.get_taskmanagers(start_time=yesterday)
    assert len(result2) == 1
    assert result2[0]["name"] == "taskmanager2"
    assert str(result2[0]["taskmanager_id"]) == "22222222-2222-2222-2222-222222222222"

    result3 = dataspace.get_taskmanagers(end_time=yesterday)
    assert len(result3) == 1
    assert result3[0]["name"] == "taskmanager1"
    assert str(result3[0]["taskmanager_id"]) == "11111111-1111-1111-1111-111111111111"

    result4 = dataspace.get_taskmanagers(taskmanager_name="taskmanager1", end_time=yesterday)
    assert len(result4) == 1
    assert result4[0]["name"] == "taskmanager1"
    assert str(result4[0]["taskmanager_id"]) == "11111111-1111-1111-1111-111111111111"

    result5 = dataspace.get_taskmanagers(taskmanager_name="taskmanager2", start_time=yesterday)
    assert len(result5) == 1
    assert result5[0]["name"] == "taskmanager2"
    assert str(result5[0]["taskmanager_id"]) == "22222222-2222-2222-2222-222222222222"

    result6 = dataspace.get_taskmanagers(
        taskmanager_name="taskmanager2", start_time=yesterday, end_time=two_years_future
    )
    assert len(result6) == 1
    assert result6[0]["name"] == "taskmanager2"
    assert str(result6[0]["taskmanager_id"]) == "22222222-2222-2222-2222-222222222222"

    result7 = dataspace.get_taskmanagers(start_time=yesterday, end_time=two_years_future)
    assert len(result7) == 1
    assert result7[0]["name"] == "taskmanager2"
    assert str(result7[0]["taskmanager_id"]) == "22222222-2222-2222-2222-222222222222"

def test_get_taskmanagers_not_exist(dataspace):  # noqa: F811
    """Do I error out when asking for garbage"""
    last_year = str(datetime.datetime.now() - datetime.timedelta(days=365))
    two_years_future = str(datetime.datetime.now() + datetime.timedelta(days=730))

    result = dataspace.get_taskmanagers(taskmanager_name="no_such_task_manager")
    assert result == []

    result = dataspace.get_taskmanagers(start_time=two_years_future)
    assert result == []

    result = dataspace.get_taskmanagers(end_time=last_year, start_time=two_years_future)
    assert result == []

def test_store_taskmanager(dataspace):  # noqa: F811
    """Can we make new entries"""
    primary_key = dataspace.store_taskmanager(
        name="taskmanager3",
        taskmanager_id="00000003-0003-0003-0003-000000000003",
    )
    assert primary_key > 1

def test_get_last_generation_id(dataspace):  # noqa: F811
    """Can we get the last generation id by name or name and uuid"""
    result1 = dataspace.get_last_generation_id(taskmanager_name="taskmanager1")
    assert result1 == 1
    result1 = dataspace.get_last_generation_id(
        taskmanager_name="taskmanager1",
        taskmanager_id="11111111-1111-1111-1111-111111111111",
    )
    assert result1 == 1

    result2 = dataspace.get_last_generation_id(
        taskmanager_name="taskmanager2",
        taskmanager_id="22222222-2222-2222-2222-222222222222",
    )
    assert result2 == 2

def test_get_last_generation_id_not_exist(dataspace):  # noqa: F811
    """Does it error out if we ask for a bogus taskmanager?"""
    with pytest.raises((KeyError, NoResultFound)):
        dataspace.get_last_generation_id(taskmanager_name="no_such_task_manager")
    with pytest.raises((KeyError, NoResultFound)):
        dataspace.get_last_generation_id(
            taskmanager_name="no_such_task_manager",
            taskmanager_id="11111111-1111-1111-1111-111111111111",
        )

def test_get_header(dataspace):  # noqa: F811
    """Can we fetch a header?"""
    result = dataspace.get_header(
        taskmanager_id=1,
        generation_id=1,
        key="my_test_key",
    )

    assert result[0] == "11111111-1111-1111-1111-111111111111"
    assert result[1] == 1
    assert result[2] == 1
    assert result[3] == "my_test_key"
    assert result[7] == "module"

def test_get_header_not_exist(dataspace):  # noqa: F811
    """Does it error out if we ask for a bogus header?"""
    with pytest.raises((KeyError, NoResultFound)):
        dataspace.get_header(
            taskmanager_id=100,
            generation_id=1,
            key="my_test_key",
        )

    with pytest.raises((KeyError, NoResultFound)):
        dataspace.get_header(
            taskmanager_id=100,
            generation_id=10,
            key="my_test_key",
        )

    with pytest.raises((KeyError, NoResultFound)):
        dataspace.get_header(
            taskmanager_id=100,
            generation_id=1,
            key="no_such_key_exists",
        )

def test_get_metadata(dataspace):  # noqa: F811
    """Can we fetch a metadata element?"""
    result = dataspace.get_metadata(
        taskmanager_id=1,
        generation_id=1,
        key="my_test_key",
    )

    assert result[0] == "11111111-1111-1111-1111-111111111111"
    assert result[1] == 1
    assert result[2] == 1
    assert result[3] == "my_test_key"

def test_get_metadata_not_exist(dataspace):  # noqa: F811
    """Does it error out if we ask for a bogus metadata element?"""
    with pytest.raises((KeyError, NoResultFound)):
        dataspace.get_metadata(
            taskmanager_id=100,
            generation_id=1,
            key="my_test_key",
        )

    with pytest.raises((KeyError, NoResultFound)):
        dataspace.get_metadata(
            taskmanager_id=100,
            generation_id=11111111,
            key="my_test_key",
        )

    with pytest.raises((KeyError, NoResultFound)):
        dataspace.get_metadata(
            taskmanager_id=100,
            generation_id=1,
            key="no_such_key_exists",
        )

def test_get_dataproducts(dataspace):  # noqa: F811
    """Can we get the dataproducts by uuid and uuid with key"""
    result1 = dataspace.get_dataproducts(taskmanager_id=1)

    assert len(result1) == 2
    assert result1[0] == {
        "taskmanager_id": 1,
        "generation_id": 1,
        "key": "my_test_key",
        "value": b"my_test_value",
    }
    assert result1[1] == {
        "taskmanager_id": 1,
        "generation_id": 1,
        "key": "a_test_key",
        "value": b"a_test_value",
    }

    result2 = dataspace.get_dataproducts(taskmanager_id="2", key="other_test_key")

    assert result2 == [
        {
            "taskmanager_id": 2,
            "generation_id": 2,
            "key": "other_test_key",
            "value": b"other_test_value",
        },
    ]

def test_get_dataproducts_not_exist(dataspace):  # noqa: F811
    """Does it error out if we ask for bogus information?"""
    result = dataspace.get_dataproducts(taskmanager_id=100)

    assert result == []
    result = dataspace.get_dataproducts(taskmanager_id=2, key="no_such_key")
    assert result == []

def test_get_dataproduct(dataspace):  # noqa: F811
    """Can we get the dataproduct by uuid with key"""
    result2 = dataspace.get_dataproduct(
        taskmanager_id=2,
        generation_id=2,
        key="other_test_key",
    )

    assert result2 == b"other_test_value"

def test_get_datablock(dataspace):  # noqa: F811
    """Can we get the datablock content"""
    result2 = dataspace.get_datablock(
        taskmanager_id=2,
        generation_id=2,
    )

    assert result2 == {"other_test_key": b"other_test_value"}

def test_get_dataproduct_not_exist(dataspace):  # noqa: F811
    """Does it error out if we ask for bogus information?"""
    with pytest.raises((KeyError, NoResultFound)):
        dataspace.get_dataproduct(
            taskmanager_id=2,
            generation_id=2,
            key="no_such_key",
        )

def test_insert(dataspace):  # noqa: F811
    """Can we insert new elements"""
    primary_key = dataspace.store_taskmanager("taskmanager3", "33333333-3333-3333-3333-333333333333")
    assert primary_key > 1

    header = Header(primary_key)
    metadata = Metadata(primary_key)
    dataspace.insert(
        primary_key,
        1,
        "sample_test_key",
        b"sample_test_value",
        header,
        metadata,
    )

    result1 = dataspace.get_dataproducts(taskmanager_id=primary_key)

    assert result1 == [
        {
            "key": "sample_test_key",
            "taskmanager_id": primary_key,
            "generation_id": 1,
            "value": b"sample_test_value",
        }
    ]

    result2 = dataspace.get_dataproducts(taskmanager_id=primary_key, key="sample_test_key")

    assert result1 == result2

def test_update(dataspace):  # noqa: F811
    """Do updates work as expected"""
    metadata_row = dataspace.get_metadata(
        taskmanager_id=1,
        generation_id=1,
        key="my_test_key",
    )
    header_row = dataspace.get_header(
        taskmanager_id=1,
        generation_id=1,
        key="my_test_key",
    )
    dataspace.update(
        taskmanager_id=1,
        generation_id=1,
        key="my_test_key",
        value=b"I changed IT",
        header=Header(
            header_row[0],
            create_time=header_row[4],
            expiration_time=header_row[5],
            scheduled_create_time=header_row[6],
            creator=header_row[7],
            schema_id=header_row[8],
        ),
        metadata=Metadata(
            metadata_row[0],
            state=metadata_row[4],
            generation_id=metadata_row[2],
            generation_time=metadata_row[5],
            missed_update_count=metadata_row[6],
        ),
    )

    result1 = dataspace.get_dataproduct(
        taskmanager_id=1,
        generation_id=1,
        key="my_test_key",
    )

    assert result1 == b"I changed IT"

def test_update_bad(dataspace):  # noqa: F811
    """Do updates fail to work on bogus taskmanager as expected"""
    metadata_row = dataspace.get_metadata(
        taskmanager_id=1,
        generation_id=1,
        key="my_test_key",
    )
    header_row = dataspace.get_header(
        taskmanager_id=1,
        generation_id=1,
        key="my_test_key",
    )
    with pytest.raises(Exception):
        dataspace.update(
            taskmanager_id=100,
            generation_id=1,
            key="my_test_key",
            value=b"I changed IT",
            header=Header(
                header_row[0],
                create_time=header_row[4],
                expiration_time=header_row[5],
                scheduled_create_time=header_row[6],
                creator=header_row[7],
                schema_id=header_row[8],
            ),
            metadata=Metadata(
                metadata_row[0],
                state=metadata_row[4],
                generation_id=metadata_row[2],
                generation_time=metadata_row[5],
                missed_update_count=metadata_row[6],
            ),
        )

def test_duplicate_datablock(dataspace):  # noqa: F811
    """Can we duplicate taskmanager1 and all its entries"""
    result1 = dataspace.get_last_generation_id(
        taskmanager_name="taskmanager1",
        taskmanager_id="11111111-1111-1111-1111-111111111111",
    )
    assert result1 == 1

    result1 = dataspace.get_dataproducts(
        taskmanager_id=1,
    )
    assert len(result1) == 2

    dataspace.duplicate_datablock(1, 1, 90)

    result1 = dataspace.get_last_generation_id(
        taskmanager_name="taskmanager1",
        taskmanager_id="11111111-1111-1111-1111-111111111111",
    )
    assert result1 == 90


# This is a sample code for using Line Search for training DNN using MNIST data
# Implemented by Shigeng Sun, Sept 2022
# Requires PyTorch, SDLS.py and optimizer.py from Pytorch
# Requires matplotlib
# Under active development,
# tested and working for SGD without momentum

import torch
from torch import nn
from torch.utils.data import DataLoader
from torchvision import datasets
from torchvision.transforms import ToTensor
from SDLS import SDLS
import matplotlib.pyplot as plt

# Download training data from open datasets.
training_data = datasets.FashionMNIST(
    root="data",
    train=True,
    download=True,
    transform=ToTensor(),
)

# Download test data from open datasets.
test_data = datasets.FashionMNIST(
    root="data",
    train=False,
    download=True,
    transform=ToTensor(),
)

batch_size = 128
lr = .8      # initial learning rate
tau = 1.5   # lr retraction ratio
c_0 = 0.1   # if rho is below this, lr gets divided    by tau
# initialize SDLS memory buffers and memory lengths
mem_len = 100
rho_list  = []
lr_list   = []
loss_list = []
Trn_accy  = []
i = 0

# Create data loaders.
train_dataloader = DataLoader(training_data, batch_size=batch_size)
test_dataloader  = DataLoader(test_data, batch_size=batch_size)

for X, y in test_dataloader:
    print(f"Shape of X [N, C, H, W]: {X.shape}") # batch, ? , height of image, width of image
    print(f"Shape of y: {y.shape} {y.dtype}")    # label
    break

# Get cpu or gpu device for training.
device = "cuda" if torch.cuda.is_available() else "cpu"
print(f"Using {device} device")

# Define model
class NeuralNetwork(nn.Module):
    def __init__(self):
        super(NeuralNetwork, self).__init__()
        self.flatten = nn.Flatten()
        self.linear_relu_stack = nn.Sequential(
            nn.Linear(28*28, 512),
            nn.ReLU(),#nn.ReLU(),
            nn.Linear(512, 512),
            nn.ReLU(), #nn.ReLU(),Sigmoid
            nn.Linear(512, 10)
        )

    def forward(self, x):
        x = self.flatten(x)
        logits = self.linear_relu_stack(x)
        return logits

model = NeuralNetwork().to(device)
print(model)

loss_fn = nn.CrossEntropyLoss()
optimizer = SDLS(model.parameters(), lr=lr, am = c_0)

def train(dataloader, model, loss_fn, optimizer):
    tau = 2   # lr retraction ratio
    c_0 = 0.4 # if rho is below this, lr gets divided by tau
    size = len(dataloader.dataset)
    model.train()
    i = 0
    for batch, (X, y) in enumerate(dataloader):
        X, y = X.to(device), y.to(device)
        (Xt , yt) = next(iter(dataloader))
        Xt, yt = Xt.to(device), yt.to(device)
        pred = model(X)
        loss = loss_fn(pred, y)
        optimizer.zero_grad()
        loss.backward()
        grad_norm = 0
        var_norm = 0
        dimdim = 0

        for _ , param in model.named_parameters():
            dimdim += torch.numel(param.grad)
            grad_norm += (torch.norm(param.grad))**2
        lr = optimizer.param_groups[0]['lr']

        # closure implementation specifies sample consistency

        # note if use Xt and Yt in the implementation, we construct sample inconsistent line search.
        # If sample inconsistent, may consider relaxing the line search in the optimizer.
        def closure():
            optimizer.zero_grad()
            #output = model(Xt)
            output = model(X)
            #loss = loss_fn(output, yt)
            loss  = loss_fn(output, y)
            loss.backward()
            return loss

        rho, lr = optimizer.step(closure,grad_norm,c_0,tau)
        lr_list.append(lr)
        print('lr:',lr, 'rho:', rho)

        if batch % 10 == 0:
            loss, current = loss.item(), batch * len(X)
            loss_list.append(loss)
            print(f"loss: {loss:>7f}  [{current:>5d}/{size:>5d}]")
        def test_t(dataloader, model, loss_fn):
            size = len(dataloader.dataset)
            num_batches = len(dataloader)
            model.eval()
            test_loss, correct = 0, 0
            with torch.no_grad():
                for X, y in dataloader:
                    X, y = X.to(device), y.to(device)
                    pred = model(X)
                    test_loss += loss_fn(pred, y).item()
                    correct += (pred.argmax(1) == y).type(torch.float).sum().item()
            test_loss /= num_batches
            correct /= size
            return correct
        i += 1
        if i%100 == 0:
            Trn_accy.append(test_t(dataloader, model, loss_fn))

def test(dataloader, model, loss_fn):
    size = len(dataloader.dataset)
    num_batches = len(dataloader)
    model.eval()
    test_loss, correct = 0, 0
    with torch.no_grad():
        for X, y in dataloader:
            X, y = X.to(device), y.to(device)
            pred = model(X)
            test_loss += loss_fn(pred, y).item()
            correct += (pred.argmax(1) == y).type(torch.float).sum().item()
    test_loss /= num_batches
    correct   /= size
    print(f"Test Error: \n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \n")

epochs = 5
for t in range(epochs):
    print(f"Epoch {t+1}\n-------------------------------")
    train(train_dataloader, model, loss_fn, optimizer)
    test(test_dataloader, model, loss_fn)
    grad_norm = 0
    total_norm = 0
    for p in model.parameters():
        param_norm = p.grad.data.norm(2)
        total_norm += param_norm.item() ** 2
    total_norm = total_norm ** (1. / 2)
    print(total_norm)

fig, (ax1, ax2,ax3) = plt.subplots(3)
ax1.plot(Trn_accy)
ax1.set_title('training accuracy')
ax2.plot(loss_list)
ax2.set_title('stochastic loss')
ax3.plot(lr_list)
ax3.set_title('learning rate')
txt="stepsize is",lr , "; batch size is", batch_size

plt.figtext(0.5, 0.01, txt, wrap=True, horizontalalignment='center', fontsize=12)
plt.show()
print("Done!")


#%%
from sqlalchemy import create_engine
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import Column, String, Date, Integer
from sqlalchemy.orm import sessionmaker
from tqdm import tqdm
import csv
from datetime import datetime as dt

#%%
engine = create_engine('postgresql+psycopg2://jon:postgres@localhost/headlines')
conn = engine.connect()
Base = declarative_base()

#%%

year = '2008'

class Headline(Base):
    __tablename__ = f'headlines_{year}'

    id = Column(Integer, primary_key = True)
    date = Column(Date)
    headline = Column(String(255))

Session = sessionmaker(bind=engine)
session = Session()
Base.metadata.create_all(conn)

#%%
filepath = f'resources/reuters-newswire-{year}.v5.csv'

num_lines = sum(1 for line in open(filepath,'r'))

with open(filepath, newline = '', encoding='iso_8859_1') as csvfile:
    reader = csv.reader(csvfile, delimiter=',')
    id_ = 1
    i = 1

    headers = next(reader, None)
    for row in tqdm(reader, total = num_lines):
        if i % 100 == 0:
            i = 0
            session.commit()

        session.add(Headline
            (
                id = id_,
                date = dt.strptime(row[0], '%Y%m%d%H%M'),
                headline = row[1])
            )
        i += 1
        id_ += 1

#%%