import os, inspect, datetime
from qgis.PyQt import uic, QtCore, QtGui
from qgis.PyQt.QtWidgets import QWidget

from ...base3.qgswidgets.lamia_form_deficiency import BaseDeficiencyTool
from .lamia_form_observation import BaseConstructionsiteObservationTool
from Lamia.api.libslamia.lamiareport.lamiareport import ReportCore

from Lamia.qgisiface.iface.qgiswidget.tools.form_subwidgets.subwidget_lidchooser import (
    LidChooserWidget,
)

# from Lamia.qgisiface.iface.qgiswidget.tools.form_subwidgets.subwidget_createsubfeature import (
#     CreateSubFeatureWidget,
# )

class BaseConstructionsiteDeficiencyTool(BaseDeficiencyTool):

    PREPROTOOLNAME = "deficiencypv"
    tooltreewidgetSUBCAT = QtCore.QCoreApplication.translate(
        "base3_cs", "Proces verbal"
    )

    def __init__(self, **kwargs):
        super(BaseConstructionsiteDeficiencyTool, self).__init__(**kwargs)

    def initMainToolWidget(self):
        if self.dbase.variante in [None, "Lamia"]:
            self.toolwidgetmain = UserUI()
            self.formtoolwidgetconfdictmain = {
                "deficiency": {
                    "linkfield": "id_deficiency",
                    "widgets": {
                        #'groupedesordre': self.toolwidgetmain.comboBox_groupedes,
                        # 'deficiencyobservatortype': self.toolwidgetmain.comboBox_detecteur,
                        # 'detecteur_com': self.toolwidgetmain.lineEdit_detecteur,
                    },
                },
                "object": {
                    "linkfield": "id_object",
                    "widgets": {
                        # 'comment': self.toolwidgetmain.lineEdit_detecteur,
                    },
                },
            }

            # self.tabWidget.removeTab(0)
            # self.SKIP_LOADING_UI = True
            self.TABLEFILTERFIELD = {"deficiencycategory": "PVE"}
            self.toolwidgetmain.stackedWidget.setCurrentIndex(1)
            self.dbasechildwdgfield = []
            self.instancekwargs["parentwidget"] = self
            self.temppropertieswdgOBSERVATION = BaseConstructionsiteObservationTool(
                **self.instancekwargs
            )
            self.temppropertieswdgOBSERVATION.TABLEFILTERFIELD = {
                "observationcategory": "PVA"
            }
            # self.temppropertieswdgOBSERVATION.SKIP_LOADING_UI = True
            self.temppropertieswdgOBSERVATION.tooltreewidgetSUBCAT = "Procès verbal"
            # temppropertieswdgOBSERVATION.tooltreewidgetSUBCAT = displayname
            # exec('self.propertieswdgOBSERVATION{} = temppropertieswdgOBSERVATION'.format(observationcategory))
            # exec('self.dbasechildwdgfield.append(self.propertieswdgOBSERVATION{} )'.format(observationcategory))
            self.dbasechildwdgfield.append(self.temppropertieswdgOBSERVATION)

            # # * lamiawdg
            # self.createobswdg = CreateSubFeatureWidget(
            #     self, self.temppropertieswdgOBSERVATION
            # )
            # self.lamiawidgets.append(self.createobswdg)

        else:
            self.unloadWidgetinToolTree()
            self.loadWidgetinToolTree = lambda: None
            # self.toolTreeWidgetCurrentItemChanged = lambda: None

    def magicFunction(self):
        self.featureSelected()
        self.addGPSPoint()
        self.saveFeature()
        for wdgobservation in self.dbasechildwdgfield:
            if hasattr(wdgobservation, "OBSTYPE") and wdgobservation.OBSTYPE == "NCA":
                wdgobservation.featureSelected()
                wdgobservation.saveFeature()

    # def postInitFeatureProperties(self, feat):
    def postSelectFeature(self):
        super().postSelectFeature()

    """
    def postSaveFeature(self, savedfeaturepk=None):
        self.formutils.createNewSubFeature(
            self.temppropertieswdgOBSERVATION, savedfeaturepk
        )
    """

    def printWidget(self):

        # create finelname
        pdfdirectory = os.path.join(self.dbase.dbaseressourcesdirectory, "Print")
        if not os.path.isdir(pdfdirectory):
            os.mkdir(pdfdirectory)

        currentid = self.dbase.getValuesFromPk(
            "deficiency", "id_deficiency", self.currentFeaturePK
        )

        date = str(datetime.datetime.now().strftime("%Y%m%d_%H%M%S"))

        filename = str(currentid) + "_deficiency_" + date + ".pdf"

        pdffilename = os.path.join(pdfdirectory, filename)

        # choose phaseA report or phase C report
        sql = (
            "SELECT observationcategory FROM  observation_now WHERE observation_now.lid_deficiency = "
            + str(currentid)
        )
        sql = self.dbase.updateQueryTableNow(sql)
        res = self.dbase.query(sql)

        if (
            res is not None
            and len(res) > 0
            and not self.dbase.utils.isAttributeNull(res[0][0])
        ):
            if res[0][0] == "PVA":
                reporttype = "TRAMprocesverbalmiseadisposition"
            """
            elif res[0][0][0:2] == "NC":
                sql = "SELECT id_observation FROM observation_now WHERE observation_now.typeobservation = 'NCB'"
                sql += " AND Observation_now.lid_desordre = " + str(currentid)
                sql = self.dbase.updateQueryTableNow(sql)
                res = self.dbase.query(sql)
                if self.dbase.variante in [None, "Lamia"]:
                    if (
                        res is not None
                        and len(res) > 0
                        and not self.dbase.utils.isAttributeNull(res[0][0])
                    ):
                        reportype = "TRAMnonconformite"
                    else:
                        reportype = "TRAMnonconformitephaseA"
                elif self.dbase.variante in ["Orange"]:
                    print("*********ORANGEnonconformitephaseA")
                    reportype = "ORANGEnonconformitephaseA"
            """
        else:
            return

        # run report
        self.reporttool = ReportCore(
            dbaseparser=self.dbase, messageinstance=self.mainifacewidget.connector
        )
        self.reporttool.runReport(
            destinationfile=pdffilename,
            reportconffilename=reporttype,
            pklist={0: [self.currentFeaturePK]},
        )

class UserUI(QWidget):
    def __init__(self, parent=None):
        super(UserUI, self).__init__(parent=parent)
        uipath = os.path.join(os.path.dirname(__file__), "lamia_form_deficiency_ui.ui")
        uic.loadUi(uipath, self)


import os
import json
import pandas as pd
import win32com.client
import logging
from calendar_operations import add_to_calendar_with_earliest_start_end, remove_from_calendar

# Configure the logging system
logging.basicConfig(
    level=logging.DEBUG,
    format="%(asctime)s %(levelname)s: %(message)s",
    datefmt="%Y-%m-%d %H:%M:%S",
)

def load_config():
    script_dir = os.path.dirname(os.path.abspath(__file__))
    script_dir = os.path.dirname(__file__)
    config_path = os.path.join(script_dir, "config.json")
    with open(config_path, "r") as f:
        return json.load(f)

if __name__ == "__main__":
    logging.info("Starting main application.")

    try:
        config = load_config()
    except FileNotFoundError as e:
        logging.error("Config file not found!")
        exit(1)
    except json.JSONDecodeError as e:
        logging.error("Error decoding JSON config!")
        exit(1)

    xlsx_file = config.get("excel_file", "./16DayOct.xlsx")
    try:
        df = pd.read_excel(xlsx_file)
    except FileNotFoundError as e:
        logging.error(f"Excel file {xlsx_file} not found!")
        exit(1)

    # Log DataFrame details
    logging.debug(f"Shape of the DataFrame before filters: {df.shape}")
    logging.debug(f"Data types in the DataFrame before filters: {df.dtypes}")

    print("Debug: Head of DataFrame after reading Excel:")
    print(df.head())
    logging.debug(f"DataFrame Head: {df.head()}")

    logging.info("Applying filters.")
    df = df[~df["Function Room"].isin(config["exclude_function_room"])]
    df = df[~df["Event Type"].isin(config["exclude_event_type"])]
    df = df[df["Event Type"].isin(config["include_event_type"])]

    # Log DataFrame details after filters
    logging.debug(f"Shape of the DataFrame after filters: {df.shape}")
    logging.debug(f"Data types in the DataFrame after filters: {df.dtypes}")

    # Initialize the Outlook object
    outlook = win32com.client.Dispatch("Outlook.Application").GetNamespace("MAPI")
    root_folder = outlook.GetDefaultFolder(9)

    # Initialize the specific calendars
    calendars = {}
    for folder in root_folder.Folders:
        if folder.Name in ["Definite", "Tentative", "Prospect"]:
            calendars[folder.Name] = folder

    if not calendars:
        logging.error("Could not find required calendars.")
        exit(1)

    action = (
        input("Would you like to 'Add' or 'Remove' events? (Enter 'Add' or 'Remove'): ")
        .strip()
        .lower()
    )

    if action == "add":
        logging.info("Adding events to calendars.")
        add_to_calendar_with_earliest_start_end(df, calendars)
    elif action == "remove":
        logging.info("Removing events from calendars.")
        remove_from_calendar(df, calendars, outlook)
    else:
        logging.error("Invalid action choice.")
        print("Invalid choice. Please enter 'Add' or 'Remove'.")


# -*- coding: utf-8 -*-
from odoo import api, models, fields, registry
import json
import ast
from odoo.tools import DEFAULT_SERVER_DATETIME_FORMAT
import odoo
import logging

_logger = logging.getLogger(__name__)

class pos_cache_database(models.Model):
    _name = "pos.cache.database"
    _description = "Management POS database"
    _rec_name = "res_id"

    res_id = fields.Char('Res Id')
    res_model = fields.Char('Res Model')
    deleted = fields.Boolean('Deleted', default=0)
    updated_date = fields.Datetime('Last updated date', readonly=1)

    @api.model
    def create(self, vals):
        vals['updated_date'] = fields.Datetime.now()
        return super(pos_cache_database, self).create(vals)

    @api.model
    def write(self, vals):
        vals['updated_date'] = fields.Datetime.now()
        return super(pos_cache_database, self).write(vals)

    @api.model
    def get_datas_backend_modified(self, updated_date):
        records = self.sudo().search([('updated_date', '>', updated_date)])
        results = []
        if records:
            for record in records:
                try:
                    val = self.env[record.res_model].browse(int(record.res_id)).get_data()
                    if record.deleted:
                        val['deleted'] = True
                        val['id'] = int(record.res_id)
                    val['write_date'] = record.updated_date
                    results.append(val)
                except:
                    val['deleted'] = True
                    val['id'] = int(record.res_id)
                    val['write_date'] = record.updated_date
                    results.append(val)
        return results

    @api.model
    def get_stock_datas(self, location_id, product_need_update_onhand=[]):
        values = {}
        product_object = self.env['product.product'].sudo()
        if not product_need_update_onhand:
            datas = product_object.with_context({'location' : location_id}).search_read(
                [('type', '=', 'product'), ('available_in_pos', '=', True)], ['qty_available'])
        else:
            datas = product_object.with_context({'location' : location_id}).search_read(
                [('id', 'in', product_need_update_onhand)],
                ['name', 'qty_available', 'default_code'])
        for data in datas:
            values[data['id']] = data['qty_available']
        return values

    @api.multi
    def get_fields_by_model(self, model_name):
        params = self.env['ir.config_parameter'].sudo().get_param(model_name)
        if not params:
            list_fields = self.env[model_name].fields_get()
            fields_load = []
            for k, v in list_fields.items():
                if v['type'] not in ['one2many', 'binary']:
                    fields_load.append(k)
            return fields_load
        else:
            params = ast.literal_eval(params)
            return params.get('fields', [])

    @api.multi
    def get_domain_by_model(self, model_name):
        params = self.env['ir.config_parameter'].sudo().get_param(model_name)
        if not params:
            return []
        else:
            params = ast.literal_eval(params)
            return params.get('domain', [])

    @api.model
    def insert_data(self, datas, model, first_install=False):
        if type(model) == list:
            return False
        all_fields = self.env[model].fields_get()
        version_info = odoo.release.version_info[0]
        if version_info == 12:
            if all_fields:
                for data in datas:
                    for field, value in data.items():
                        if field == 'model':
                            continue
                        if all_fields[field] and all_fields[field]['type'] in ['date', 'datetime'] and value:
                            data[field] = value.strftime(DEFAULT_SERVER_DATETIME_FORMAT)
        if first_install:
            for data in datas:
                self.create({
                    'res_id': str(data['id']),
                    'res_model': model,
                })
        else:
            for data in datas:
                last_caches = self.search([('res_id', '=', str(data['id'])), ('res_model', '=', model)])
                if last_caches:
                    last_caches.write({
                        'updated_date': fields.Datetime.now(),
                    })
                else:
                    self.create({
                        'res_id': str(data['id']),
                        'res_model': model,
                    })
        return True

    def sync_to_pos(self, data):
        if data['model'] == 'product.product':
            data['price'] = data['list_price']
        sessions = self.env['pos.session'].sudo().search([
            ('state', '=', 'opened')
        ])
        self.insert_data([data], data['model'])
        for session in sessions:
            self.env['bus.bus'].sendmany(
                [[(self.env.cr.dbname, 'pos.sync.data', session.user_id.id), data]])
        return True

    @api.model
    def remove_record(self, data):
        self.search([('res_id', '=', str(data['id'])), ('res_model', '=', data['model'])]).write({
            'deleted': True
        })
        sessions = self.env['pos.session'].sudo().search([
            ('state', '=', 'opened')
        ])
        data['deleted'] = True
        for session in sessions:
            self.env['bus.bus'].sendmany(
                [[(self.env.cr.dbname, 'pos.sync.data', session.user_id.id), data]])
        return True

    @api.model
    def save_parameter_models_load(self, model_datas):
        for model_name, value in model_datas.items():
            self.env['ir.config_parameter'].sudo().set_param(model_name, value)
        return True


import tweepy
from get_all_tickers import get_tickers as gt

# This function scrapes the users you type in and attempts to grab their top count tweets and process them and add them to the maps.
def scrapeUsers(mapOfStocks, allStocks):
    # List of the accounts we want to scrap
    usernames = []
    # how many tweets you want
    count = 150

    for name in usernames:
        try:
            #Create query method using parameters
            tweets = tweepy.Cursor(api.user_timeline, id=name).items(count)
            # Grab the important info from the persons tweets
            tweets_list = [[tweet.created_at, tweet.id, tweet.text] for tweet in tweets]

            for tweet in tweets_list:
                tweetDescp = tweet.text
                addStocks(tweetDescp, mapOfStocks, allStocks)

        except BaseException as e:
            print("failed on_status,", str(e))

# This function needs to add any stock tickers seen in sentnce into into the mapOfStocks
# sentence- The sentence that needs to parsed which possibly contains any stock tickers
# mapOfStocks - The mapping between stock tickers and the number of times they appear
# allstocks- A map of all stock tickers
def addStocks(sentence, mapOfStocks, allstocks):
    # Split the sentence into words
    splits = sentence.split()
    # Loop through each word
    for word in splits:
        # If the word is a ticker then add it to the mapOfStocks
        if allstocks.get(word) != None:
            if mapOfStocks.get(word) == None:
                mapOfStocks.update({word: 1})
            else:
                mapOfStocks.update({word: mapOfStocks.get(word) + 1})

# converts the stockslist into a map
def listToMap(stockslist):
    allStocks = {}
    for i in stockslist:
        allStocks.update({i: 1})
    return allStocks

# Set up authentication
consumer_key = "xxx"
consumer_secret = "xxx"
access_token = "xxx"
access_token_secret = "xxxx"

auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
api = tweepy.API(auth, wait_on_rate_limit=True)

# Create the map and get all the stocks from the csv
mapOfStocks = {}
stockslist = gt.get_tickers()
allStocks = listToMap(stockslist)

scrapeUsers(mapOfStocks, allStocks)

# sort the values of the mapofstocks and go through and print all of them in order
sortedVals = sorted(mapOfStocks.values())
seen = {}

#initialize the seen array
for key in mapOfStocks.keys():
    seen.update({key: False})

# Print the stuff out
for i in sortedVals:
    for key in mapOfStocks.keys():
        if mapOfStocks.get(key) == i and seen.get(key) == False:
            print(key + ": " + str(i))
            seen.update({key: True})
            break


import time
import numpy as np
import sys
import random
import os
import warnings

warnings.filterwarnings("ignore", category=UserWarning)

import torch
from torch.utils.tensorboard import SummaryWriter
import torch.distributed as dist
import torch.multiprocessing as mp
import torch.utils.data.distributed
from tqdm import tqdm

sys.path.append('./')

from data_process.kitti_dataloader import create_train_dataloader, create_val_dataloader
from models.model_utils import create_model, make_data_parallel, get_num_parameters
from utils.train_utils import create_optimizer, create_lr_scheduler, get_saved_state, save_checkpoint
from utils.train_utils import reduce_tensor, to_python_float, get_tensorboard_log
from utils.misc import AverageMeter, ProgressMeter
from utils.logger import Logger
from config.train_config import parse_train_configs
from evaluate import evaluate_mAP

def main():
    configs = parse_train_configs()

    # Re-produce results
    if configs.seed is not None:
        random.seed(configs.seed)
        np.random.seed(configs.seed)
        torch.manual_seed(configs.seed)
        torch.backends.cudnn.deterministic = True
        torch.backends.cudnn.benchmark = False

    if configs.gpu_idx is not None:
        print('You have chosen a specific GPU. This will completely disable data parallelism.')

    if configs.dist_url == "env://" and configs.world_size == -1:
        configs.world_size = int(os.environ["WORLD_SIZE"])

    configs.distributed = configs.world_size > 1 or configs.multiprocessing_distributed

    if configs.multiprocessing_distributed:
        configs.world_size = configs.ngpus_per_node * configs.world_size
        mp.spawn(main_worker, nprocs=configs.ngpus_per_node, args=(configs,))
    else:
        main_worker(configs.gpu_idx, configs)

def main_worker(gpu_idx, configs):
    configs.gpu_idx = gpu_idx
    configs.device = torch.device('cpu' if configs.gpu_idx is None else 'cuda:{}'.format(configs.gpu_idx))

    if configs.distributed:
        if configs.dist_url == "env://" and configs.rank == -1:
            configs.rank = int(os.environ["RANK"])
        if configs.multiprocessing_distributed:
            # For multiprocessing distributed training, rank needs to be the
            # global rank among all the processes
            configs.rank = configs.rank * configs.ngpus_per_node + gpu_idx

        dist.init_process_group(backend=configs.dist_backend, init_method=configs.dist_url,
                                world_size=configs.world_size, rank=configs.rank)
        configs.subdivisions = int(64 / configs.batch_size / configs.ngpus_per_node)
    else:
        configs.subdivisions = int(64 / configs.batch_size)

    configs.is_master_node = (not configs.distributed) or (
            configs.distributed and (configs.rank % configs.ngpus_per_node == 0))

    if configs.is_master_node:
        logger = Logger(configs.logs_dir, configs.saved_fn)
        logger.info('>>> Created a new logger')
        logger.info('>>> configs: {}'.format(configs))
        tb_writer = SummaryWriter(log_dir=os.path.join(configs.logs_dir, 'tensorboard'))
    else:
        logger = None
        tb_writer = None

    # model
    model = create_model(configs)

    # load weight from a checkpoint
    if configs.pretrained_path is not None:
        assert os.path.isfile(configs.pretrained_path), "=> no checkpoint found at '{}'".format(configs.pretrained_path)
        model.load_state_dict(torch.load(configs.pretrained_path))
        if logger is not None:
            logger.info('loaded pretrained model at {}'.format(configs.pretrained_path))

    # resume weights of model from a checkpoint
    if configs.resume_path is not None:
        assert os.path.isfile(configs.resume_path), "=> no checkpoint found at '{}'".format(configs.resume_path)
        model.load_state_dict(torch.load(configs.resume_path))
        if logger is not None:
            logger.info('resume training model from checkpoint {}'.format(configs.resume_path))

    # Data Parallel
    model = make_data_parallel(model, configs)

    # Make sure to create optimizer after moving the model to cuda
    optimizer = create_optimizer(configs, model)
    lr_scheduler = create_lr_scheduler(optimizer, configs)
    configs.step_lr_in_epoch = True if configs.lr_type in ['multi_step'] else False

    # resume optimizer, lr_scheduler from a checkpoint
    if configs.resume_path is not None:
        utils_path = configs.resume_path.replace('Model_', 'Utils_')
        assert os.path.isfile(utils_path), "=> no checkpoint found at '{}'".format(utils_path)
        utils_state_dict = torch.load(utils_path, map_location='cuda:{}'.format(configs.gpu_idx))
        optimizer.load_state_dict(utils_state_dict['optimizer'])
        lr_scheduler.load_state_dict(utils_state_dict['lr_scheduler'])
        configs.start_epoch = utils_state_dict['epoch'] + 1

    if configs.is_master_node:
        num_parameters = get_num_parameters(model)
        logger.info('number of trained parameters of the model: {}'.format(num_parameters))

    if logger is not None:
        logger.info(">>> Loading dataset & getting dataloader...")
    # Create dataloader
    train_dataloader, train_sampler = create_train_dataloader(configs)
    if logger is not None:
        logger.info('number of batches in training set: {}'.format(len(train_dataloader)))

    if configs.evaluate:
        val_dataloader = create_val_dataloader(configs)
        precision, recall, AP, f1, ap_class = evaluate_mAP(val_dataloader, model, configs, None)
        print('Evaluate - precision: {}, recall: {}, AP: {}, f1: {}, ap_class: {}'.format(precision, recall, AP, f1,
                                                                                ap_class))
        print('mAP {}'.format(AP.mean()))
        return

    for epoch in range(configs.start_epoch, configs.num_epochs + 1):
        if logger is not None:
            logger.info('{}'.format('*-' * 40))
            logger.info('{} {}/{} {}'.format('=' * 35, epoch, configs.num_epochs, '=' * 35))
            logger.info('{}'.format('*-' * 40))
            logger.info('>>> Epoch: [{}/{}]'.format(epoch, configs.num_epochs))

        if configs.distributed:
            train_sampler.set_epoch(epoch)
        # train for one epoch
        train_one_epoch(train_dataloader, model, optimizer, lr_scheduler, epoch, configs, logger, tb_writer)
        if not configs.no_val:
            val_dataloader = create_val_dataloader(configs)
            print('number of batches in val_dataloader: {}'.format(len(val_dataloader)))
            precision, recall, AP, f1, ap_class = evaluate_mAP(val_dataloader, model, configs, logger)
            val_metrics_dict = {
                'precision': precision.mean(),
                'recall': recall.mean(),
                'AP': AP.mean(),
                'f1': f1.mean(),
                'ap_class': ap_class.mean()
            }
            if tb_writer is not None:
                tb_writer.add_scalars('Validation', val_metrics_dict, epoch)

        # Save checkpoint
        if configs.is_master_node and ((epoch % configs.checkpoint_freq) == 0):
            model_state_dict, utils_state_dict = get_saved_state(model, optimizer, lr_scheduler, epoch, configs)
            save_checkpoint(configs.checkpoints_dir, configs.saved_fn, model_state_dict, utils_state_dict, epoch)

        if not configs.step_lr_in_epoch:
            lr_scheduler.step()
            if tb_writer is not None:
                tb_writer.add_scalar('LR', lr_scheduler.get_lr()[0], epoch)

    if tb_writer is not None:
        tb_writer.close()
    if configs.distributed:
        cleanup()

def cleanup():
    dist.destroy_process_group()

def train_one_epoch(train_dataloader, model, optimizer, lr_scheduler, epoch, configs, logger, tb_writer):
    batch_time = AverageMeter('Time', ':6.3f')
    data_time = AverageMeter('Data', ':6.3f')
    losses = AverageMeter('Loss', ':.4e')

    progress = ProgressMeter(len(train_dataloader), [batch_time, data_time, losses],
                             prefix="Train - Epoch: [{}/{}]".format(epoch, configs.num_epochs))

    num_iters_per_epoch = len(train_dataloader)

    # switch to train mode
    model.train()
    start_time = time.time()
    for batch_idx, batch_data in enumerate(tqdm(train_dataloader)):
        data_time.update(time.time() - start_time)
        _, imgs, targets = batch_data
        global_step = num_iters_per_epoch * (epoch - 1) + batch_idx + 1

        batch_size = imgs.size(0)

        targets = targets.to(configs.device, non_blocking=True)
        imgs = imgs.to(configs.device, non_blocking=True)
        total_loss, outputs = model(imgs, targets)

        # For torch.nn.DataParallel case
        if (not configs.distributed) and (configs.gpu_idx is None):
            total_loss = torch.mean(total_loss)

        # compute gradient and perform backpropagation
        total_loss.backward()
        if global_step % configs.subdivisions == 0:
            optimizer.step()
            # Adjust learning rate
            if configs.step_lr_in_epoch:
                lr_scheduler.step()
                if tb_writer is not None:
                    tb_writer.add_scalar('LR', lr_scheduler.get_lr()[0], global_step)
            # zero the parameter gradients
            optimizer.zero_grad()

        if configs.distributed:
            reduced_loss = reduce_tensor(total_loss.data, configs.world_size)
        else:
            reduced_loss = total_loss.data
        losses.update(to_python_float(reduced_loss), batch_size)
        # measure elapsed time
        # torch.cuda.synchronize()
        batch_time.update(time.time() - start_time)

        if tb_writer is not None:
            if (global_step % configs.tensorboard_freq) == 0:
                tensorboard_log = get_tensorboard_log(model)
                tb_writer.add_scalar('avg_loss', losses.avg, global_step)
                for layer_name, layer_dict in tensorboard_log.items():
                    tb_writer.add_scalars(layer_name, layer_dict, global_step)

        # Log message
        if logger is not None:
            if (global_step % configs.print_freq) == 0:
                logger.info(progress.get_message(batch_idx))

        start_time = time.time()

if __name__ == '__main__':
    try:
        main()
    except KeyboardInterrupt:
        try:
            cleanup()
            sys.exit(0)
        except SystemExit:
            os._exit(0)


from gensim.models import Word2Vec
import numpy as np

model = Word2Vec.load('Word2Vec_1.model') #load model

#print(model.wv.vocab.keys())
#print(model.wv.similar_by_word("เอเลี่ยน"))

max_value = 0
min_value = 0
for i in model.wv.vocab.keys():
    max_value = max(np.max(model.wv.get_vector(i)),max_value)
    min_value = min(np.min(model.wv.get_vector(i)),min_value)
    print(min_value,"-",max_value)


# coding:utf-8
import torch

from transformers import BartTokenizer, BartForConditionalGeneration, BartConfig
hf_config = BartConfig.from_pretrained('facebook/bart-large')
hf_config.force_bos_token_to_be_generated = True
tok = BartTokenizer.from_pretrained("facebook/bart-large")
hf_model = BartForConditionalGeneration.from_pretrained('facebook/bart-large', config=hf_config)

# fsq bart base
# from fairseq.models.bart import BARTModel
# bart = BARTModel.from_pretrained('/nfs/Desktop/data/pretrained-model/bart.large', checkpoint_file='model.pt')

# mask_token_id = bart.task.source_dictionary.indices["<mask>"]
# mask_token_weight_fairseq = bart.model.encoder.embed_tokens.weight[mask_token_id].detach()

# mask_token_id_hf = tok.mask_token_id
# mask_token_weight_hf = hf_model.model.encoder.embed_tokens.weight[mask_token_id_hf].detach()

# print((mask_token_weight_hf - mask_token_weight_fairseq).abs().max())

# input_string = " My dog is <mask>"
# labels_string = " My dog is cute"
input_string = "Rudolph Agnew <mask> 55 years old and former chairman of <mask> ated <mask> <mask> PLC , was named a nonexecutive director of this British industrial conglomerate ."
labels_string = "Rudolph Agnew , 55 years old and former chairman of Consolidated Gold Fields PLC , was named a nonexecutive director of this British industrial conglomerate ."

input_ids = tok(input_string, return_tensors="pt").input_ids
labels = tok(labels_string, return_tensors="pt").input_ids
# print("labels", labels)
# print(tok.batch_decode(labels.tolist()))
decoder_input_ids = hf_model.prepare_decoder_input_ids_from_labels(labels)
print("input_ids", input_ids)
print(tok.batch_decode(input_ids.tolist()))
print("dec_inp_ids", decoder_input_ids)
print(tok.batch_decode(decoder_input_ids.tolist()))
print("labels", labels)
print(tok.batch_decode(labels.tolist()))
loss = hf_model(input_ids=input_ids, decoder_input_ids=decoder_input_ids, labels=labels)[0]
print("Loss1", loss)

# from transformers import BartTokenizer, BartForConditionalGeneration
# tokenizer = BartTokenizer.from_pretrained('facebook/bart-large')
# model = BartForConditionalGeneration.from_pretrained('facebook/bart-large')

# TXT = "My friends are <mask> but they eat too many carbs."
# input_ids = tokenizer([TXT], return_tensors='pt')['input_ids']
# print(input_ids)
# logits = model(input_ids).logits

# masked_index = (input_ids[0] == tokenizer.mask_token_id).nonzero().item()
# probs = logits[0, masked_index].softmax(dim=0)
# values, predictions = probs.topk(5)
# print(tokenizer.decode(predictions).split())


import numpy as np
from Environment import Environment, play
from Util import _printoptions
from itertools import product

class FrozenLake(Environment):
    def __init__(self, lake, slip, max_steps, seed=None):
        """
        lake: A matrix that represents the lake. For example:
         lake =  [['&', '.', '.', '.'],
                  ['.', '#', '.', '#'],
                  ['.', '.', '.', '#'],
                  ['#', '.', '.', '$']]
        slip: The probability that the agent will slip
        max_steps: The maximum number of time steps in an episode
        seed: A seed to control the random number generator (optional)
        """

        # start (&), frozen (.), hole (#), goal ($)
        self.lake = np.array(lake)
        self.lake_flat = self.lake.reshape(-1)

        self.slip = slip

        n_states = self.lake.size + 1
        n_actions = 4

        pi = np.zeros(n_states, dtype=float)
        pi[np.where(self.lake_flat == '&')[0]] = 1.0

        self.absorbing_state = n_states - 1

        # TODO:
        # Call parent constructor
        Environment.__init__(self, n_states, n_actions, max_steps, pi, seed)

        # Up, left, down, right (corresponding to w, a, s, d)
        self.actions = [(-1, 0), (0, -1), (1, 0), (0, 1)]

        # Matrix containing rewards for TAKING AN ACTION at a state
        self.reward_map = np.zeros(self.lake.shape, dtype=np.float)
        # Set goal state to 1
        self.reward_map[np.where(self.lake == '$')] = 1

        # Matrix indicating where the absorbing states are (holes & goal states are 1, others are 0)
        self.abs_states = np.zeros(self.lake.shape, dtype=np.float)
        # Set goal state to 1
        self.abs_states[np.where(self.lake == '$')] = 1
        self.abs_states[np.where(self.lake == '#')] = 1

        # Helpers for conversions from indices to states (coordinates) and states (coordinates) to indices
        self.state_idx_to_coords = list(product(range(self.reward_map.shape[0]), range(self.reward_map.shape[1])))
        self.coords_to_state_idx = {s: i for (i, s) in enumerate(self.state_idx_to_coords)}

        # Precompute probabilities for transitions
        # self.probabilities = np.zeros((self.n_states, self.n_states, self.n_actions))
        self.probabilities = {state: {action: [] for action in range(n_actions)} for state in range(n_states)}

        def increment(row, col, action):
            """
            Helper function for generating transition probability matrix.
            Makes sure that our agents don't leave the playing field.
            :param row: Current row
            :param col: Current column
            :param action: Incoming action
            :return: New row and column coordinates
            """
            # Boundary checks to make sure that agents don't leave the field
            if action == 0:  # up
                row = max(row - 1, 0)
            elif action == 1:  # left
                col = max(col - 1, 0)
            elif action == 2:  # down
                row = min(row + 1, self.lake.shape[0] - 1)
            elif action == 3:
                col = min(col + 1, self.lake.shape[1] - 1)
            return row, col

        def update_prob_matrix(row, col, action):
            """
            Helper function for generating transition probability matrix.
            Takes a current position (row, col) and an action and returns
            the new state, the reward for the state and whether the agent is done (won or fell in hole)
            :param row:
            :param col:
            :param action:
            :return:
            """
            # Get next field coordinates with boundary checks
            new_row, new_col = increment(row, col, action)
            # Convert coordinates to state ID
            new_state = self.coords_to_state_idx[(new_row, new_col)]
            # Get new coordinate type
            f_type = self.lake[row][column]
            # Check whether we reached the goal or fell in hole
            done = f_type == '$' or f_type == '#'
            # Reward = 1 if agent is at goal, 0 otherwise
            reward = float(f_type == '$')
            return new_state, reward, done

        # Generate transition probabilities
        # Adapted from the openai gym implementation
        # https://gym.openai.com/envs/FrozenLake-v0/
        # Go through all fields and
        for row in range(self.lake.shape[0]):
            for column in range(self.lake.shape[1]):
                # Get state index from coordinates
                state_idx = self.coords_to_state_idx[(row, column)]
                # Go through all actions (up, left, down, right)
                for action in range(n_actions):
                    # Get transition probabilities for current state and action
                    current_list = self.probabilities[state_idx][action]
                    # Check if field is goal or hole
                    field_type = self.lake[row][column]
                    # If goal or hole, make inescapable (probability 1.0)
                    if field_type == '$':  # goal => reward 1, done = True
                        current_list.append((1.0, state_idx, 1.0, True))
                    elif field_type == '#':  # hole => reward 0, done = True
                        current_list.append((1.0, state_idx, 0.0, True))
                    # Otherwise check where we can go from here:
                    else:
                        # Add probabilities for successful action and slips
                        for b in range(n_actions):
                            if b == action:
                                # Successful action
                                # The asterisk ('*') unpacks the return value of the update_prob_matrix() function
                                # The probability for a successful action is 1 minus the slip probability
                                # for each remaining move
                                # Note: For this to work the slip probability has to be in the range [0...1]
                                current_list.append(
                                    (1.0 - (n_actions - 1.0) * self.slip, *update_prob_matrix(row, column, b))
                                )
                            else:
                                # Slip :(
                                current_list.append((self.slip, *update_prob_matrix(row, column, b)))

    def step(self, action):
        # This call updates self.step as well
        state, reward, done = Environment.step(self, action)

        # Check whether agent is done
        done = (state == self.absorbing_state) or done

        return state, reward, done

    def p(self, next_state, state, action):
        # Initialise transition probability
        transition_p = 0.0
        # Get possible moves for the state and action (successful move + slips)
        possible_moves = self.probabilities[state][action]
        for proba, next_s, reward, done in possible_moves:
            if next_state == next_s:
                transition_p += proba

        return transition_p

    def r(self, next_state_idx, state_idx, action):
        possible_actions = self.probabilities[state_idx][action]
        for proba, next_state, reward, done in possible_actions:
            if next_state == next_state_idx:
                return reward

    def render(self, policy=None, value=None):
        if policy is None:
            lake = np.array(self.lake_flat)

            if self.state < self.absorbing_state:
                lake[self.state] = '@'

            print(lake.reshape(self.lake.shape))
        else:
            # UTF-8 arrows look nicer, but cannot be used in LaTeX
            # https://www.w3schools.com/charsets/ref_utf_arrows.asp
            actions = ['^', '<', '_', '>']

            print('Lake:')
            print(self.lake)

            print('Policy:')
            policy = np.array([actions[a] for a in policy[:-1]])
            print(policy.reshape(self.lake.shape))

            print('Value:')
            with _printoptions(precision=3, suppress=True):
                print(value[:-1].reshape(self.lake.shape))

if __name__ == "__main__":
    seed = 0

    # Small lake
    lake = [['&', '.', '.', '.'],
            ['.', '#', '.', '#'],
            ['.', '.', '.', '#'],
            ['#', '.', '.', '$']]

    env = FrozenLake(lake, slip=0.1, max_steps=16, seed=seed)
    play(env)


from django.http import HttpResponse
from django.shortcuts import redirect, render
from base.models import Message, Room, Topic
from django.db.models import Q

# Custom Service object imports
# from base.services.RoomService import room_service_obj

# Custom Service imports
from base.services.RoomService import RoomService

# Create your views here.

def home(request):
    q = request.GET.get("q") if request.GET.get("q") is not None else ""
    rooms = Room.objects.filter(
        Q(topic__name__icontains=q) | Q(name__icontains=q) | Q(description__icontains=q)
    )
    page = request.GET.get("page", 1)
    rooms_obj = RoomService.pagination(request, page, rooms)
    rooms_count = Room.objects.count()
    topics = Topic.objects.all()[:5]
    all_messages = Message.objects.all()
    room_data = []

    for room in rooms_obj:
        participants_count = room.participants.count()
        room_data.append(
            {"room": room, "participants_count": participants_count}
        )

    context = {
        "topics": topics,
        "rooms_data": room_data,
        "rooms": rooms_obj,
        "rooms_count": rooms_count,
        "all_messages": all_messages,
    }

    return render(request, "base/home.html", context)


# -*- coding: utf-8 -*-
"""
Created on Fri Apr 15 08:42:27 2022

@author: Admin
"""

from Analytics import eurostats_loader
import pandas as pd
import numpy as np
from statsmodels.tsa.deterministic import CalendarFourier, DeterministicProcess
from scipy.signal import periodogram
from sklearn.linear_model  import LinearRegression
from sklearn.ensemble import GradientBoostingRegressor, RandomForestRegressor
import matplotlib as mpl
import matplotlib.pyplot as plt

mpl.rcParams['axes.prop_cycle'] = mpl.cycler(color=["#0645ad", "#800020"])

#%% Create Data
Weekly_Deaths = eurostats_loader()
Weekly_Deaths = Weekly_Deaths[Weekly_Deaths.index.year >= 2008]

Weekly_Deaths_train = Weekly_Deaths[Weekly_Deaths.index.year < 2019]
Weekly_Deaths_val = Weekly_Deaths[Weekly_Deaths.index.year == 2019]
Weekly_Deaths_test = Weekly_Deaths[Weekly_Deaths.index.year > 2019]

#%% Periodogram of Series
fs = pd.Timedelta("1Y") / pd.Timedelta("1W")
f, Pxx = periodogram(Weekly_Deaths_train.Value, fs=fs, detrend="linear", scaling="spectrum")
# The Power Spectrum Pxx indicates the share of each frequency of the variance of the Time Series.
print(np.sum(Pxx) == np.var(Weekly_Deaths_train.Value))
plt.plot(f, Pxx)
plt.xlabel('Frequency [Year]')
plt.ylabel('Power Spectrum [V**2]')
plt.show()

#%% Modeling of Time Series

fourierA = CalendarFourier(freq="A", order=3)

y_train = Weekly_Deaths_train.Value

dp = DeterministicProcess(index=y_train.index.to_period("W"),
                          constant=True,               # dummy feature for bias (y-intercept)
                          order=1,                     # trend (order 1 means linear)
                          additional_terms=[fourierA],) # annual seasonality (fourier)
X_train = dp.in_sample()

val_index = Weekly_Deaths_val.index
X_val = dp.out_of_sample(len(val_index), val_index)

#%% Fit Models and Plot Results.

Models = [LinearRegression(fit_intercept=False),
          GradientBoostingRegressor(max_depth=1)]

num_models = len(Models)
fig, axes = plt.subplots(num_models,2,layout="constrained",sharex="col",figsize = (15,8))

for i,model in enumerate(Models):

    model.fit(X_train,y_train)
    y_pred = pd.Series(model.predict(X_train).reshape(-1), index=y_train.index)
    y_val = pd.Series(model.predict(X_val).reshape(-1), index=val_index)

    mse = np.mean(np.sqrt((y_val-Weekly_Deaths_val.Value)**2))

    ax_train = axes[i,0]
    #ax_train.set_title(type(model).__name__)
    y_pred.plot(ax=ax_train)
    Weekly_Deaths_train.Value.plot(ax=ax_train)

    ax_val = axes[i,1]
    ax_val.axes.annotate(f"MSE: {mse:.2f}", xy=(0.8,0.9), xycoords='axes fraction', fontsize=10)
    y_val.plot(ax=ax_val)
    Weekly_Deaths_val.Value.plot(ax=ax_val)

#%% Calculate Excess Deaths with best model.

y_comb_train = Weekly_Deaths[Weekly_Deaths.index.year < 2020]

dp = DeterministicProcess(index=y_comb_train.index.to_period("W"),
                          constant=True,               # dummy feature for bias (y-intercept)
                          order=1,                     # trend (order 1 means linear)
                          additional_terms=[fourierA],  # annual seasonality (fourier)
                          drop=True,)                  # drop terms to avoid collinearity

X_comb_train = dp.in_sample()

model = GradientBoostingRegressor(max_depth=1)
model.fit(X_comb_train, y_comb_train)
y_comb_pred = pd.Series(model.predict(X_comb_train).reshape(-1), index=y_comb_train.index)

test_index = Weekly_Deaths_test.index
X_test = dp.out_of_sample(len(test_index), test_index)
y_test = pd.Series(model.predict(X_test).reshape(-1), index=test_index)
excess_deaths = np.sum(Weekly_Deaths_test.Value-y_test)

fig_test, ax_test = plt.subplots()
ax_test.axes.annotate(f"Estimated Excess Deaths: {excess_deaths:.0f}", xy=(0.55,0.9), xycoords='axes fraction', fontsize=8)
y_test.plot(ax=ax_test)
Weekly_Deaths_test.Value.plot(ax=ax_test)


from typing import List

from stx import app, logger
from stx.components import (
    Component, Composite, CodeBlock, Table, Image,
    FunctionCall, CustomText, Layout)
from stx.components import ListBlock, Paragraph, PlainText, StyledText
from stx.components import LinkText, Literal, Figure, Section, Separator
from stx.components import ContentBox, TableOfContents, ElementReference
from stx.components import CapturedText
from stx.document import Document
from stx.outputs.html.dom import Tag
from stx.outputs.html.themes import HtmlTheme
from stx.utils.stx_error import StxError

TYPE_H_TAGS = {
    'chapter': 'h1',
    'sect1': 'h1',
    'sect2': 'h2',
    'sect3': 'h3',
    'sect4': 'h4',
    'sect5': 'h5',
}

CAPTURED_CLASS_TAGS = [
    'cite', 'ins', 'kbd', 'mark', 'q', 's',
    'samp', 'small', 'sub', 'sup', 'var',
]

def document_to_html(document: Document, theme: HtmlTheme) -> List[Tag]:
    logger.info('Serializing HTML5 book...')

    html = Tag('html')

    generate_head(document, html, theme)

    generate_body(document, html, theme)

    return [Tag('!DOCTYPE html'), html]

def generate_head(document: Document, html: Tag, theme: HtmlTheme):
    head = html.append_tag('head')

    if document.encoding:
        head.append_tag('meta', {'charset': document.encoding})

    if document.author:
        head.append_tag('meta', {
            'name': 'author',
            'content': document.author,
        })

    head.append_tag('meta', {
        'name': 'generator',
        'content': app.app_title,
    })

    if document.title:
        head.append_tag('title', text=document.title)

    for css in theme.head_styles:
        head.append_tag('style', {
            'type': 'text/css',
        }, text=css, text_literal=True)

    for stylesheet in document.stylesheets:
        head.append_tag('link', {
            'rel': 'stylesheet',
            'type': 'text/css',
            'href': stylesheet,
        })

    for js in theme.head_scripts:
        head.append_tag('script', {
            'type': 'application/javascript',
        }, text=js, text_literal=True)

def generate_body(document: Document, html: Tag, theme: HtmlTheme):
    body = html.append_tag('body', {'data-type': 'book'})

    if document.title is not None:
        body.append_tag('h1', text=document.title)

    if document.header is not None:
        header = body.append_tag('header')

        generate_component(header, document.header)

    if document.content is not None:
        generate_component(body, document.content)

    if document.footer is not None:
        footer = body.append_tag('footer')

        generate_component(footer, document.footer)

    for js in theme.body_scripts:
        body.append_tag('script', {
            'type': 'application/javascript',
        }, text=js, text_literal=True)

def generate_component(
        parent: Tag, component: Component,
        collapse_paragraph=False,
        collapse_section=False):
    if isinstance(component, Composite):
        generate_components(parent, component.components)
    elif isinstance(component, CodeBlock):
        generate_code_block(parent, component)
    elif isinstance(component, Table):
        generate_table(parent, component)
    elif isinstance(component, ListBlock):
        generate_list_block(parent, component)
    elif isinstance(component, Paragraph):
        generate_paragraph(parent, component, collapse_paragraph)
    elif isinstance(component, PlainText):
        generate_plain_text(parent, component)
    elif isinstance(component, StyledText):
        generate_styled_text(parent, component)
    elif isinstance(component, CustomText):
        generate_custom_text(parent, component)
    elif isinstance(component, LinkText):
        generate_link_text(parent, component)
    elif isinstance(component, CapturedText):
        generate_captured_text(parent, component)
    elif isinstance(component, Literal):
        generate_literal(parent, component)
    elif isinstance(component, Figure):
        generate_figure(parent, component)
    elif isinstance(component, Section):
        generate_section(parent, component, collapse_section)
    elif isinstance(component, TableOfContents):
        generate_toc(parent, component)
    elif isinstance(component, Separator):
        generate_separator(parent, component)
    elif isinstance(component, ContentBox):
        generate_box(parent, component)
    elif isinstance(component, Image):
        generate_image(parent, component)
    elif isinstance(component, FunctionCall):
        generate_function_call(parent, component)
    elif isinstance(component, Layout):
        generate_layout(parent, component)
    else:
        raise NotImplementedError(f'Not implemented type: {type(component)}')

def generate_components(parent: Tag, components: List[Component]):
    for child in components:
        generate_component(parent, child)

def generate_code_block(parent: Tag, code_block: CodeBlock):
    pre = append_component_tag(parent, code_block, 'pre')

    pre.preserve_spaces = True

    pre['data-type'] = 'programlisting'

    if code_block.lang is not None:
        pre['data-code-language'] = code_block.lang

    generate_components(pre, code_block.contents)

def generate_table(parent: Tag, table: Table):
    table_tag = append_component_tag(parent, table, 'table')

    if table.caption is not None:
        caption_tag = table_tag.append_tag('caption')

        if table.number is not None:
            caption_tag['data-number'] = table.number

        generate_component(
            caption_tag, table.caption, collapse_paragraph=True)

    for row in table.rows:
        row_tag = table_tag.append_tag('tr')

        if row.header:
            cell_tag_name = 'th'
        else:
            cell_tag_name = 'td'

        for cell in row.cells:
            cell_tag = row_tag.append_tag(cell_tag_name)

            generate_component(cell_tag, cell, collapse_paragraph=True)

def generate_list_block(parent: Tag, list_block: ListBlock):
    if list_block.ordered:
        list_tag_name = 'ol'
    else:
        list_tag_name = 'ul'

    list_tag = append_component_tag(parent, list_block, list_tag_name)

    for item in list_block.items:
        item_tag = list_tag.append_tag('li')

        generate_component(item_tag, item, collapse_paragraph=True)

def generate_paragraph(parent: Tag, paragraph: Paragraph, collapse: bool):
    if not collapse:
        p_tag = append_component_tag(parent, paragraph, 'p')
    else:
        p_tag = parent

    generate_components(p_tag, paragraph.contents)

def generate_plain_text(parent: Tag, plain_text: PlainText):
    for ref in plain_text.get_refs():
        parent.append_tag('a', {'id': ref}, text='')

    parent.append_text(plain_text.content)

def generate_styled_text(parent: Tag, styled_text: StyledText):
    if styled_text.style == 'strong':
        tag_name = 'strong'
    elif styled_text.style == 'emphasized':
        tag_name = 'em'
    elif styled_text.style == 'code':
        tag_name = 'code'
    elif styled_text.style == 'deleted':
        tag_name = 'del'
    else:
        tag_name = None

    if tag_name is not None:
        styled_tag = append_component_tag(parent, styled_text, tag_name)

        generate_components(styled_tag, styled_text.contents)
    else:
        if styled_text.style == 'double-quote':
            parent.append_literal('“')
            generate_components(parent, styled_text.contents)
            parent.append_literal('”')
        elif styled_text.style == 'single-quote':
            parent.append_literal('‘')
            generate_components(parent, styled_text.contents)
            parent.append_literal('’')
        else:
            raise StxError(
                f'Not supported style: {styled_text.style}.',
                styled_text.location)

def generate_custom_text(parent: Tag, styled_text: CustomText):
    styled_tag = append_component_tag(parent, styled_text, 'span')

    if styled_text.custom_style is not None:
        styled_tag['class'] = styled_text.custom_style

    generate_components(styled_tag, styled_text.contents)

def generate_link_text(parent: Tag, link_text: LinkText):
    a_tag = append_component_tag(parent, link_text, 'a')

    if link_text.is_internal():
        a_tag['href'] = f'#{link_text.reference}'
        a_tag['data-type'] = 'xref'
    elif link_text.is_external():
        a_tag['href'] = link_text.reference

    if link_text.invalid:
        a_tag['data-status'] = 'invalid'

    generate_components(a_tag, link_text.contents)

def generate_captured_text(parent: Tag, captured: CapturedText):
    if captured.class_ in CAPTURED_CLASS_TAGS:
        cap_tag = append_component_tag(parent, captured, captured.class_)
    else:
        cap_tag = append_component_tag(parent, captured, 'span')

        if captured.class_:
            cap_tag['class'] = captured.class_

    generate_components(cap_tag, captured.contents)

def generate_literal(parent: Tag, literal: Literal):
    parent.append_literal(literal.text)

def generate_figure(parent: Tag, figure: Figure):
    figure_tag = append_component_tag(parent, figure, 'figure')

    generate_component(figure_tag, figure.content)

    caption_tag = figure_tag.append_tag('figcaption')

    if figure.number is not None:
        caption_tag['data-number'] = figure.number

    generate_component(caption_tag, figure.caption, collapse_paragraph=True)

def generate_section(parent: Tag, section: Section, collapse: bool):
    if not collapse:
        section_tag = append_component_tag(parent, section, 'section')
    else:
        section_tag = parent

    if section.type is not None:
        section_tag['data-type'] = section.type

    if section.type in TYPE_H_TAGS:
        h_tag_name = TYPE_H_TAGS[section.type]
    else:
        level = section.level

        if level < 1:
            level = 1
        elif level > 6:
            level = 6

        h_tag_name = f'h{level}'

    h_tag = section_tag.append_tag(h_tag_name)

    if section.number is not None:
        h_tag['data-number'] = section.number

    generate_component(h_tag, section.heading, collapse_paragraph=True)

    generate_component(section_tag, section.content)

def generate_toc(parent: Tag, toc: TableOfContents):
    nav_tag = append_component_tag(parent, toc, 'nav')
    nav_tag['data-type'] = 'toc'

    if toc.title is not None:
        nav_tag.append_tag('h1', text=toc.title)

    generate_toc_elements(nav_tag, toc.elements)

def generate_toc_elements(parent: Tag, elements: List[ElementReference]):
    if len(elements) == 0:
        return

    list_tag = parent.append_tag('ol')

    for element in elements:
        item_tag = list_tag.append_tag('li')

        a_tag = item_tag.append_tag('a', {
            'href': f'#{element.reference}',
            'data-type': 'xref',
        }, text=element.title)

        if element.number is not None:
            a_tag['data-number'] = element.number

        generate_toc_elements(item_tag, element.elements)

def generate_separator(parent: Tag, separator: Separator):
    sep = append_component_tag(parent, separator, 'hr')
    sep['data-level'] = separator.level

def generate_box(parent: Tag, box: ContentBox):
    box_tag = append_component_tag(parent, box, 'div')

    if box.style is not None:
        box_tag['data-type'] = box.style

    generate_component(box_tag, box.content, collapse_section=True)

def generate_image(parent: Tag, image: Image):
    img_tag = parent.append_tag('img')
    img_tag['src'] = image.src

    if image.alt:
        img_tag['alt'] = image.alt

def generate_function_call(parent: Tag, call: FunctionCall):
    if call.result is None:
        raise StxError(
            f'Function {call.key} was not processed.', call.location)

    generate_component(parent, call.result)

def generate_layout(parent: Tag, layout: Layout):
    container = parent.append_tag('div', {
        'data-type': 'layout',
        'data-dir': layout.direction,
    })

    for component in layout.components:
        item = container.append_tag('div', {
            'data-type': 'layout-item',
        })

        generate_component(item, component)

def append_component_tag(
        parent: Tag, component: Component, tag_name: str) -> Tag:
    for other_id in reversed(component.get_other_refs()):
        parent.append_tag('a', {'id': other_id}, text='')

    tag = parent.append_tag(tag_name)

    tag_id = component.get_main_ref()

    if tag_id is not None:
        tag['id'] = tag_id

    return tag


import cv2
import numpy as np
import matplotlib.pyplot as plt

image = cv2.imread("/Users/masayakinefuchi/labo/imagesensing2/ImageSensing2/clahe2.png")
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)

M = np.ones(image.shape, dtype="uint8") * 130

added_image = cv2.add(image, M)

subtracted_image = cv2.subtract(image, M)

plt.subplot(1, 3, 1)
plt.title('original')
plt.xticks([]), plt.yticks([])
plt.imshow(image)

plt.subplot(1, 3, 2)
plt.title('added_image')
plt.xticks([]), plt.yticks([])
plt.imshow(added_image)
plt.imsave("addimage.png",added_image)

plt.subplot(1, 3, 3)
plt.title('subtracted_image')
plt.xticks([]), plt.yticks([])
plt.imshow(subtracted_image)

plt.show()


import argparse

class MonoCalibrator:

    def __init__(
        self,
        camera_model,
        pattern,
        rows,
        cols,
        size,
    ):
        self.camera_model = camera_model
        self.pattern = pattern
        self.rows = rows
        self.cols = cols
        self.size = size

def parse_args():
    parser = argparse.ArgumentParser(description='mono calibration')
    parser.add_argument('--camera_model', type=str, default='pinhole')
    parser.add_argument('--pattern', type=str, default='chessboard')
    parser.add_argument('--rows', type=int, default=6)
    parser.add_argument('--cols', type=int, default=6)
    parser.add_argument('--size', type=float, default=0.025)
    parser.add_argument('--device', type=str, default='/dev/video0')
    args = parser.parse_args()
    return args

# so many things to do

def main():
    args = parse_args()

    # check parameters
    # camera_model must be "pinhole" or "fisheye"
    assert args.camera_model in ['pinhole', 'fisheye']
    # pattern type must be "chessboard" or "charuco"
    assert args.pattern in ['chessboard', 'charuco']
    # size must be a float
    assert isinstance(args.size, float)
    # device_name must be a string
    assert isinstance(args.device, str)

    # print args
    print('args:')
    print(f'camera_model: {args.camera_model}')
    print(f'pattern: {args.pattern}')
    print(f'rows: {args.rows}')
    print(f'cols: {args.cols}')
    print(f'size: {args.size}')
    print(f'device: {args.device}')

    # start calibration

if __name__ == '__main__':
    main()


from fastapi.testclient import TestClient
from main import app
from queries.accounts import AccountQueries
from authenticator import authenticator

client = TestClient(app)
test_user = {
    "id": 42,
    "email": "emailll@email.email",
    "username": "dummydata",
    "avatar_img": "12321312",
    "score": 30,
}

class TestAccountQueries:
    def get_all_accounts(self):
        return [test_user]

def accounts_override():
    return test_user

def test_get_all_accounts():
    app.dependency_overrides[AccountQueries] = TestAccountQueries
    app.dependency_overrides[
        authenticator.try_get_current_account_data
    ] = accounts_override
    response = client.get("/accounts")
    assert response.json() == {"accounts": [test_user]}
    assert response.status_code == 200
    app.dependency_overrides = {}

def test_init():
    assert 1 == 1


import datetime
from setups import dp
from helpers import add_user, add_new_search, filter_search_locations
from aiogram.types import Message, CallbackQuery, ReplyKeyboardRemove, InlineKeyboardMarkup, InlineKeyboardButton
from aiogram.dispatcher.filters.state import State, StatesGroup
from aiogram.dispatcher.filters import Text
from aiogram.dispatcher import FSMContext
import logging
from helpers import add_user, add_new_search, RapidapiHelper, cancel_search_by_user, update_city_name, update_city_id,\
    get_value, set_value, build_hotels_list, sort_hotels_by_score, filter_image_list, cancel_search_by_error,\
    update_history_data, commands_desc, succes_end_search, filter_hotels_by_price, slice_list, sort_hotels_by_ds,\
    sort_hotels_by_price_and_score
from keyboards import main_menu_keybord, choice_keyboard, cancel_keyboard
import json
from aiogram_calendar import SimpleCalendar, simple_cal_callback
import emoji

class SearchStates(StatesGroup):
    get_city_name = State()
    get_check_in_date = State()
    get_check_out_date = State()
    get_adults_count = State()
    get_children = State()
    get_min_price = State()
    get_max_price = State()
    get_result_count = State()
    user_choice = State()

sort_orders = {
    "lowprice": "PRICE_LOW_TO_HIGH",
    "highprice": "топ самых дорогих отелей в городе",
    "bestdeal": "DISTANCE"
}

NO_PHOTO_URL = 'https://thumbs.dreamstime.com/b/'\
    'no-image-available-icon-photo-camera-flat-vector-illustration-132483141.jpg'

distance_dict = {
    "kilometer": "км.",
    "mile": "мл."
}

@dp.message_handler(Text(equals="отмена", ignore_case=True), state="*")
async def cancel_search(message: Message, state: FSMContext) -> None:
    """Отмена поиска."""
    data = await state.get_data()
    logger = logging.getLogger(__name__)
    logger.error(
        f"Пользователь {message.from_user.id} отменил поиск"
    )
    cancel_search_by_user(search_id=get_value(data, "search id"))
    await state.finish()
    await message.answer(
        "Поиск отменен",
        reply_markup=main_menu_keybord
    )

@dp.message_handler(commands=["lowprice", "highprice", "bestdeal"], state=None)
async def start_search(message: Message, state: FSMContext) -> None:
    """Начало поиска топа дешевых отелей"""
    logger = logging.getLogger(__name__)
    logger.info(
        f"Пользователь {message.from_user.id} начал поиск."
    )
    # получаем идентификатор пользователя.
    user_id = add_user(id=message.from_user.id)
    await SearchStates.get_city_name.set()
    # заполняем словарь со служебной информацией о поиске
    command = message.text[1:]
    search_id = add_new_search(user_id=user_id, kind=command)
    helper = RapidapiHelper.get_helper()
    data = {
        "user": {
            "tg id": message.from_user.id,
            "user id": user_id
        },
        "location": {
            "city name": "",
            "city id": ""
        },
        "settlers": {
            "adults": None,
            "children": []
        },
        "dates": {
            "check in": "",
            "check out": ""
        },
        "command": {
            "cmd name": command
        },
        "search data": {
            "search id": search_id
        },
        "price": {
            "min price": None,
            "max price": None
        },
        "count": {
            "value": 5
        }
    }
    await state.set_data(data)
    await message.answer(
        f"Вы хотите найти {commands_desc[command]}. Я помогу вам.\nВведите название города:",
        reply_markup=cancel_keyboard
    )

@dp.message_handler(state=SearchStates.get_city_name)
async def get_city_name(message: Message, state: FSMContext) -> None:
    """Получение названия города и первичный поиск."""
    city_name = message.text
    data = await state.get_data()
    set_value(data, "city name", city_name)
    await state.set_data(data)
    update_city_name(search_id=get_value(data, "search id"), city_name=city_name)
    await message.answer(
        f"Поиск данных о городе '{city_name}'"
    )
    helper = RapidapiHelper.get_helper()
    metadata = get_value(data, "metadata")
    result = helper.search_location(
        name=city_name
    )
    # проверим, что получили верные данные
    if result and result["rc"] == "OK":
        # обработка полученных данных
        city_data = result["sr"]
        city_list = filter_search_locations(city_data)
        # если в списке только 1 город, запоминаем его идентификатор и идем дальше
        if len(city_list) == 1:
            city_id = get_value(city_list[0], 'gaiaId')
            set_value(data, "city id", city_id)
            await state.set_data(data)
            await message.answer(
                "Запомнил."
            )
            await SearchStates.next()
            await message.answer(
                "Выберите дате въезда:",
                reply_markup= await SimpleCalendar().start_calendar()
            )
        elif  len(city_list) == 0:
            await message.answer(
                f"Я не нашел никаких данных о городе '{city_name}'. Уточните название и повторите поиск.",
                reply_markup=main_menu_keybord
            )
            await state.finish()
            cancel_search_by_error(search_id=get_value(data, "search id"))
        else:
            # попросим уточнить город у пользователя
            buttons_list = [
                InlineKeyboardButton(
                    text=get_value(i_city, "fullName"),
                    callback_data=get_value(i_city, "gaiaId")
                )
                for i_city in city_list
            ]
            city_kbd = InlineKeyboardMarkup(
                row_width=1
            )
            city_kbd.add(*buttons_list)
            await message.answer(
                "Уточните город:",
                reply_markup=city_kbd
            )

@dp.callback_query_handler(simple_cal_callback.filter(), state=SearchStates.get_check_in_date)
async def get_check_in_date(callback_query: CallbackQuery, callback_data: dict) -> None:
    """
    Получение даты въезда.
    """
    selected, date = await SimpleCalendar().process_selection(callback_query, callback_data)
    if selected:
        state = dp.current_state()
        data = await state.get_data()
        current_date = datetime.datetime.now()
        if date < current_date:
            await callback_query.message.answer(
                "Ошибка! Дата въезда не может быть раньше текущей даты!",
                reply_markup=ReplyKeyboardRemove()
            )
            await callback_query.message.answer(
                "Выберите дату въезда:",
                reply_markup= await SimpleCalendar().start_calendar()
            )
        else:
            check_in = date.strftime("%d.%m.%Y")
            set_value(data, "check in", check_in)
            await state.set_data(data)
            await callback_query.message.answer(
                check_in,
                reply_markup=ReplyKeyboardRemove()
            )
            await SearchStates.next()
            await callback_query.message.answer(
                "Выберите дату выезда:",
                reply_markup= await SimpleCalendar().start_calendar()
            )

@dp.callback_query_handler(simple_cal_callback.filter(), state=SearchStates.get_check_out_date)
async def get_check_out_date(callback_query: CallbackQuery, callback_data: dict) -> None:
    """
    Получение даты выезда.
    """
    state = dp.current_state()
    data = await state.get_data()
    selected, date = await SimpleCalendar().process_selection(callback_query, callback_data)
    if selected:
        check_in = datetime.datetime.strptime(get_value(data, "check in"), "%d.%m.%Y")
        current_date = datetime.datetime.now()
        if date < current_date:
            await callback_query.message.answer(
                "Ошибка! Дата выезда не может быть раньше текущей даты!",
                reply_markup=ReplyKeyboardRemove()
            )
            await callback_query.message.answer(
                "Выберите дату выезда: ",
                reply_markup= await SimpleCalendar().start_calendar()
            )
        elif date < check_in:
            await callback_query.message.answer(
                "Ошибка! Дата выезда не может быть раньше даты въезда!",
                reply_markup=ReplyKeyboardRemove()
            )
            await callback_query.message.answer(
                "Выберите дату выезда: ",
                reply_markup= await SimpleCalendar().start_calendar()
            )
        else:
            check_out = date.strftime("%d.%m.%Y")
            set_value(data, "check out", check_out)
            await state.set_data(data)
            await callback_query.message.answer(
                check_out,
                reply_markup=ReplyKeyboardRemove()
            )
            await SearchStates.next()
            await callback_query.message.answer(
                "Введите количество взрослых постояльцев:",
                reply_markup=cancel_keyboard
            )

@dp.message_handler(state=SearchStates.get_adults_count)
async def get_adults_count(message: Message, state: FSMContext) -> None:
    """
    Получение количества взрослых постояльцев.
    """
    text = message.text
    if not text.isnumeric() or text == "0":
        await message.answer(
            "Ошибка! Ожидается число выше нуля. Повторите ввод."
        )
    else:
        adults_count = int(text)
        data = await state.get_data()
        set_value(data, "adults", adults_count)
        update_history_data(
            id=get_value(data, "search id"),
            data={"adults": adults_count}
        )
        await state.set_data(data)
        await SearchStates.next()
        await message.answer(
            "Если среди постояльцев будут дети - введите возраст каждого ребенка через пробел.\nИначе введите ноль.",
            reply_markup=cancel_keyboard
        )

@dp.message_handler(state=SearchStates.get_children)
async def get_children(message: Message, state: FSMContext) -> None:
    """
    Получение возраста детей.
    """
    text = message.text
    # детей нет, идем дальше
    data = await state.get_data()
    if text == "0":
        set_value(data, "children", [])
        await SearchStates.next()
        await message.answer(
            "Введите минимальную цену номера за сутки: ",
            reply_markup=cancel_keyboard
        )
    else:
        children = []
        for i_age in text.split():
            if not i_age.isnumeric():
                await message.answer(
                    "Ошибка! Вы ввели не число."
                )
                await message.answer(
                    "Если среди постояльцев будут дети - введите возраст каждого ребенка "\
                    "через пробел.\nИначе введите ноль.",
                    reply_markup=cancel_keyboard
                )
                return
            else:
                children.append(
                    {"age": int(i_age)}
                )
        update_history_data(
            id=get_value(data, "search id"),
            data={"children": len(children)}
        )
        set_value(data, "children", children)
        await state.set_data(data)
        await SearchStates.next()
        await message.answer(
            "Введите минимальную цену номера за сутки:",
            reply_markup=cancel_keyboard
        )

@dp.message_handler(state=SearchStates.get_min_price)
async def get_min_price(message: Message, state: FSMContext) -> None:
    """
    Получение минимальной цены номера за сутки.
    """
    text = message.text
    if  not text.isnumeric():
        await message.answer(
            "Ошибка! Вы ввели не число. Повторите ввод.",
            reply_markup=cancel_keyboard
        )

    elif text == "0" or text.startswith("-"):
        await message.answer(
            "Ошибка! Цена должна быть выше нуля. Повторите ввод."
        )
    else:
        data = await state.get_data()
        set_value(data, "min price", int(text))
        await state.set_data(data)
        await SearchStates.next()
        await message.answer(
            "Введите максимальную цену номера",
            reply_markup=cancel_keyboard
        )

@dp.message_handler(state=SearchStates.get_max_price)
async def get_max_price(message: Message, state: FSMContext) -> None:
    """
    Получение максимальной цены номера за сутки.
    """
    text = message.text
    if  not text.isnumeric():
        await message.answer(
            "Ошибка! Вы ввели не число. Повторите ввод.",
            reply_markup=cancel_keyboard
        )
    elif text == "0" or text.startswith("-"):
        await message.answer(
            "Ошибка! Цена должна быть выше нуля. Повторите ввод."
        )
    else:
        data = await state.get_data()
        min_price = get_value(data, "min price")
        max_price = int(text)
        if max_price < min_price:
            await message.answer(
                "Ошибка! Максимальная цена не может быть меньше минимальной. Повторите ввод"
            )
            return
        set_value(data, "max price", max_price)
        await state.set_data(data)
        await SearchStates.next()
        await message.answer(
            "Сколько отелей показывать в ответе?",
            reply_markup=cancel_keyboard
        )

@dp.callback_query_handler(state=SearchStates.get_city_name)
async def get_city_id(query: CallbackQuery, state: FSMContext) -> None:
    """
    Получение идентификатора города.
    """
    city_id = query.data
    data = await state.get_data()
    set_value(data, "city id", city_id)
    search_id = get_value(data, "search id")
    update_city_id(search_id=search_id, city_id=city_id)
    await SearchStates.next()
    await state.set_data(data)
    await query.message.answer(
        "Выберите дату въезда: ",
        reply_markup= await SimpleCalendar().start_calendar()
    )

@dp.message_handler(state=SearchStates.get_result_count)
async def get_result_count(message: Message, state: FSMContext) -> None:
    """Получение кол-ва отелей в ответе и проверка данных поиска"""
    text = message.text
    # отрицательное число или равно нулю
    if text.startswith("-") or text == "0":
        await message.answer(
            "Ошибка! Число должно быть выше нуля. Повторите ввод.",
            reply_markup=cancel_keyboard
        )
    # введено не число
    elif not text.isnumeric():
        await message.answer(
            "Ошибка! Вы ввели не число. Повторите ввод.",
            reply_markup=cancel_keyboard
        )
    else:
        count = int(text)
        data = await state.get_data()
        set_value(data, "value", count)
        await state.set_data(data)
        # попросим проверить собранные данные перед началом поиска
        city_name = get_value(data, "city name")
        kind = commands_desc[get_value(data, "cmd name")]
        check_in = get_value(data, "check in")
        check_out = get_value(data, "check out")
        adults_count = get_value(data, "adults")
        child_count = len(get_value(data, "children"))
        min_price = get_value(data, "min price")
        max_price = get_value(data, "max price")
        msg =\
        f"Проверьте данные поиска:\n"\
        f"В {city_name} вы ищете {kind}.\n"\
        f"Постояльцы:\n"\
        f"Взрослые: {adults_count}.\n"\
        f"Дети: {child_count}\n"\
        f"Дата въезда: {check_in}.\n"\
        f"Дата выезда: {check_out}.\n"\
        f"Мин. цена: {min_price}.\n"\
        f"Макс. цена: {max_price}.\n"\
        f"Кол-во отелей в ответе: {count}.\n"\
        f"Начать поиск?"
        await SearchStates.next()
        await message.answer(
            msg,
            reply_markup=choice_keyboard
        )

@dp.message_handler(Text(equals="искать",  ignore_case=True), state=SearchStates.user_choice)
async def search_offers(message: Message, state: FSMContext) -> None:
    """Поиск предложений и вывод результатов."""
    #region поиск данных
    data = await state.get_data()
    command = get_value(data, "cmd name")
    search_id = get_value(data, "search id")
    await message.answer(
        "Ищу предложения...",
        reply_markup=ReplyKeyboardRemove()
    )
    helper = RapidapiHelper.get_helper()
    prop_data = helper.get_properties_list(
        data=data,
        sort_order=commands_desc[command]
    )
    #endregion
    #region обработка данных
    # нет данных
    if not prop_data:
        await message.answer(
            "Поисковый сервис не вернул данных. Попробуйте изменить критерии поиска или повторить позднее.",
            reply_markup=main_menu_keybord
        )
        await state.finish()
        cancel_search_by_error(
            search_id=search_id
        )
    # ошибка поиска
    elif "errors" in prop_data:
        await message.answer(
            "Поисковый сервис вернул ошибку. Попробуйте изменить критерии поиска или повторить позднее.",
            reply_markup=main_menu_keybord
        )
        await state.finish()
    # данные есть, можно работать
    else:
        # получим "сырой" список отелей
        raw_hotels_list = build_hotels_list(
            props_list=get_value(
                data=prop_data,
                key="properties"
            )
        )
        # выкинем отели, не подходящие по диапазону цен
        min_price = get_value(data, "min price")
        max_price = get_value(data, "max price")
        hotels_list_filtered_by_price = filter_hotels_by_price(
            raw_list=raw_hotels_list,
            min_price=min_price,
            max_price=max_price
        )
        # для каждой команды своя сортировка
        await message.answer(
            "Сортирую данные..."
        )
        if command == "lowprice":
            # сортировка для топа дешевых отелей
            sorted_hotels_list = sort_hotels_by_score(
                hotels_list=hotels_list_filtered_by_price
            )
        elif command == "highprice":
            # сортировка для топа дорогих отелей
            sorted_hotels_list = sort_hotels_by_price_and_score(
                hotels_list=hotels_list_filtered_by_price,
                is_reverse=True
            )
        else:
            sorted_hotels_list = sort_hotels_by_ds(
                hotels_list=hotels_list_filtered_by_price
            )
        # обрежем список до нужного размера
        count = get_value(data, "value")
        result_hotels_list = slice_list(
            raw_list=sorted_hotels_list,
            count=count
        )
    #endregion
    #region загрузка фотографий
        await message.answer(
            "Изучаю фотографии..."
        )
        for i_hotel in result_hotels_list:
            hotel_details = helper.get_hotel_details(
                id=i_hotel["id"]
            )
            if hotel_details:
                set_value(i_hotel, "address", get_value(hotel_details, "addressLine"))
                images_list = filter_image_list(
                    images_list=get_value(hotel_details, "images")
                )
                if images_list:
                    set_value(
                        i_hotel,
                        "image",
                        get_value(images_list[0], "url")
                    )
                else:
                    set_value(
                        i_hotel,
                        "image",
                        NO_PHOTO_URL
                    )
            else:
                set_value(
                    i_hotel,
                    "image",
                    NO_PHOTO_URL
                )
    #endregion
    #region вывод результата
    await message.answer(
        "Вот, что я нашел:"
    )
    for i_hotel in result_hotels_list:
        hotel_address = i_hotel["address"]
        hotel_name = i_hotel["name"]
        image = i_hotel["image"]
        score = i_hotel["score"]
        amount = i_hotel["amount"]
        distance = i_hotel["distance"]
        units = i_hotel["units"]
        text_list = [
            hotel_name,
            hotel_address,
            f"{score} {score * emoji.emojize(':star:')}",
            str(amount)
        ]
        if distance:
            text_list.append(f"{distance} {distance_dict[units.lower()]} от центра")
        msg = '\n'.join(
            text_list
        )
        await message.answer_photo(
            photo=image,
            caption=msg
        )
    succes_end_search(search_id=search_id)
    await state.finish()
    await message.answer(
        "Поиск успешно завершен.",
        reply_markup=main_menu_keybord
    )
    #endregion


import re
import ast
import json
from datetime import datetime

from SlothAI.lib.util import random_string

class MissingTemplateKey(Exception):
    def __init__(self, key: str) -> None:
        super().__init__(f"Missing template key: {key}")

class InvalidTemplateInputFields(Exception):
    def __init__(self, error: str) -> None:
        super().__init__(f"Invalid template input_fields: {error}")

class InvalidTemplateOutputFields(Exception):
    def __init__(self, error: str) -> None:
        super().__init__(f"Invalid template input_fields: {error}")

class InvalidTemplateExtras(Exception):
    def __init__(self, error: str) -> None:
        super().__init__(f"Invalid template input_fields: {error}")

class Template():
    def __init__(self, id, name, user_id, text, created, input_fields, output_fields, extras, processor) -> None:
        self.id = id
        self.name = name
        self.user_id = user_id
        self.text = text
        self.created = created
        self.input_fields = input_fields # generally, derived from text
        self.output_fields = output_fields # generally, derived from text
        self.extras = extras # generally, derived from text
        self.processor = processor # generally, derived from text

    @classmethod
    def from_dict(self, dict):
        if 'name' not in dict:
            raise MissingTemplateKey('name')
        if 'user_id' not in dict and 'uid' not in dict:
            raise MissingTemplateKey('user_id or uid')
        if 'text' not in dict:
            raise MissingTemplateKey('text')
        if 'created' not in dict:
            raise MissingTemplateKey('created')

        # updates must contain keys above and derived fields will always be
        # recomputed. This makes it harder to updated a derived field without
        # updating the text of the actual template which could cause unexpected
        # results.
        input_fields, output_fields = self.fields_from_template(dict.get('text'))
        extras = self.extras_from_template(dict.get('text'))
        processor = extras.get('processor', dict.get('processor', 'jinja2'))

        id = dict.get('id', random_string(16))
        uid = dict.get('user_id', dict.get('uid'))
        if isinstance(dict['created'], str):
            try:
                created = datetime.strptime(dict["created"], '%Y-%m-%dT%H:%M:%SZ')
            except Exception:
                created = dict['created']
        else:
            created = dict['created']

        return self(
            id=id,
            name=dict['name'],
            user_id=uid,
            text=dict['text'],
			created=created,
            input_fields=input_fields,
            output_fields=output_fields,
            extras=extras,
            processor=processor
        )

    def to_dict(self):
        if isinstance(dict['created'], str):
            created = self.created
        else:
            created = self.created.strftime('%Y-%m-%dT%H:%M:%SZ')

        return {
			"id": self.id,
			"user_id": self.user_id,
			"name": self.name,
			"text": self.text,
			"created": created,
			"input_fields": self.input_fields,
			"output_fields": self.output_fields,
			"extras": self.extras,
			"processor": self.processor,
		}

    def to_json(self) -> str:
        dict = self.to_dict()
        return json.dumps(dict, indent=4)

    @classmethod
    def from_json(self, json_str: str):
        dict = json.loads(json_str)
        return self.from_dict(dict)

    # TODO: create template model. these should go there.
    @classmethod
    def extras_from_template(self, template):
        # TODO: remove comments before processing...
        # only uses the last pattern. leaving these here for testing.
        # extras_pattern = re.compile(r'extras\s*=\s*{([\s\S]*?)}\s*', re.DOTALL)
        # extras_pattern = re.compile(r'extras\s*=\s*{([\s\S]*?)}\s*}', re.DOTALL)
        # extras_pattern = re.compile(r'extras\s*=\s*{.*?}', re.DOTALL)
        # extras_pattern = re.compile(r'extras\s*=\s*\{(?:\s*".*?"\s*:\s*".*?"\s*,?)*}', re.DOTALL)
        extras_pattern = re.compile(r'extras\s*=\s*{((?:[^{}]|{{[^{}]*}})*)}', re.DOTALL)

        extras_matches = extras_pattern.findall(template)

        try:
            extras_content = ast.literal_eval("{" + extras_matches[0] + "}")
            if not isinstance(extras_content, dict):
                raise InvalidTemplateExtras(f"extras not evaluated to dict: got {type(extras_content)}")
            return extras_content
        except Exception as ex:
            raise InvalidTemplateExtras(str(ex))

    @classmethod
    def fields_from_template(self, template):
        input_fields = []
        output_fields = []

        input_content, output_content = self.fields_text_from_template(template)

        try:
            input_fields = ast.literal_eval(input_content) if input_content else None
        except Exception as e:
            raise InvalidTemplateInputFields(str(e))

        try:
            output_fields = ast.literal_eval(output_content) if output_content else None
        except Exception as e:
            raise InvalidTemplateOutputFields(str(e))

        return input_fields, output_fields

    @classmethod
    def fields_text_from_template(self, template):
        # Regular expressions to find input and output fields in the template
        input_pattern = re.compile(r'input_fields\s*=\s*(\[.*?\])', re.DOTALL)
        output_pattern = re.compile(r'output_fields\s*=\s*(\[.*?\])', re.DOTALL)

        input_match = input_pattern.search(template)
        output_match = output_pattern.search(template)

        input_content = input_match.group(1) if input_match else None
        output_content = output_match.group(1) if output_match else None

        return input_content, output_content

    @classmethod
    def remove_fields_and_extras(self, template):
        # Remove extras definition
        extras_pattern = re.compile(r'extras\s*=\s*{([\s\S]*?)}\s*', re.DOTALL)
        template = extras_pattern.sub('', template)

        # Remove input_fields and output_fields definitions
        input_pattern = re.compile(r'input_fields\s*=\s*(\[.*?\])', re.DOTALL)
        output_pattern = re.compile(r'output_fields\s*=\s*(\[.*?\])', re.DOTALL)

        template = input_pattern.sub('', template)
        template = output_pattern.sub('', template)

        return template