import argparse
import os
import re
import time
from pathlib import Path

import torch
from torch.utils.data import DataLoader
from tqdm import tqdm

from models.vocoders.vocoder_inference import synthesis
from torch.utils.data import DataLoader
from utils.util import set_all_random_seed
from utils.util import load_config

def parse_vocoder(vocoder_dir):
    r"""Parse vocoder config"""
    vocoder_dir = os.path.abspath(vocoder_dir)
    ckpt_list = [ckpt for ckpt in Path(vocoder_dir).glob("*.pt")]
    ckpt_list.sort(key=lambda x: int(x.stem), reverse=True)
    ckpt_path = str(ckpt_list[0])
    vocoder_cfg = load_config(os.path.join(vocoder_dir, "args.json"), lowercase=True)
    vocoder_cfg.model.bigvgan = vocoder_cfg.vocoder
    return vocoder_cfg, ckpt_path

class BaseInference(object):
    def __init__(self, cfg, args):
        self.cfg = cfg
        self.args = args
        self.model_type = cfg.model_type
        self.avg_rtf = list()
        set_all_random_seed(10086)
        os.makedirs(args.output_dir, exist_ok=True)

        if torch.cuda.is_available():
            self.device = torch.device("cuda")
        else:
            self.device = torch.device("cpu")
            torch.set_num_threads(10)  # inference on 1 core cpu.

        # Load acoustic model
        self.model = self.create_model().to(self.device)
        state_dict = self.load_state_dict()
        self.load_model(state_dict)
        self.model.eval()

        # Load vocoder model if necessary
        if self.args.checkpoint_dir_vocoder is not None:
            self.get_vocoder_info()

    def create_model(self):
        raise NotImplementedError

    def load_state_dict(self):
        self.checkpoint_file = self.args.checkpoint_file
        if self.checkpoint_file is None:
            assert self.args.checkpoint_dir is not None
            checkpoint_path = os.path.join(self.args.checkpoint_dir, "checkpoint")
            checkpoint_filename = open(checkpoint_path).readlines()[-1].strip()
            self.checkpoint_file = os.path.join(
                self.args.checkpoint_dir, checkpoint_filename
            )

        self.checkpoint_dir = os.path.split(self.checkpoint_file)[0]

        print("Restore acoustic model from {}".format(self.checkpoint_file))
        raw_state_dict = torch.load(self.checkpoint_file, map_location=self.device)
        self.am_restore_step = re.findall(r"step-(.+?)_loss", self.checkpoint_file)[0]

        return raw_state_dict

    def load_model(self, model):
        raise NotImplementedError

    def get_vocoder_info(self):
        self.checkpoint_dir_vocoder = self.args.checkpoint_dir_vocoder
        self.vocoder_cfg = os.path.join(
            os.path.dirname(self.checkpoint_dir_vocoder), "args.json"
        )
        self.cfg.vocoder = load_config(self.vocoder_cfg, lowercase=True)
        self.vocoder_tag = self.checkpoint_dir_vocoder.split("/")[-2].split(":")[-1]
        self.vocoder_steps = self.checkpoint_dir_vocoder.split("/")[-1].split(".")[0]

    def build_test_utt_data(self):
        raise NotImplementedError

    def build_testdata_loader(self, args, target_speaker=None):
        datasets, collate = self.build_test_dataset()
        self.test_dataset = datasets(self.cfg, args, target_speaker)
        self.test_collate = collate(self.cfg)
        self.test_batch_size = min(
            self.cfg.train.batch_size, len(self.test_dataset.metadata)
        )
        test_loader = DataLoader(
            self.test_dataset,
            collate_fn=self.test_collate,
            num_workers=self.args.num_workers,
            batch_size=self.test_batch_size,
            shuffle=False,
        )
        return test_loader

    def inference_each_batch(self, batch_data):
        raise NotImplementedError

    def inference_for_batches(self, args, target_speaker=None):
        ###### Construct test_batch ######
        loader = self.build_testdata_loader(args, target_speaker)

        n_batch = len(loader)
        now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time()))
        print(
            "Model eval time: {}, batch_size = {}, n_batch = {}".format(
                now, self.test_batch_size, n_batch
            )
        )
        self.model.eval()

        ###### Inference for each batch ######
        pred_res = []
        with torch.no_grad():
            for i, batch_data in enumerate(loader if n_batch == 1 else tqdm(loader)):
                # Put the data to device
                for k, v in batch_data.items():
                    batch_data[k] = batch_data[k].to(self.device)

                y_pred, stats = self.inference_each_batch(batch_data)

                pred_res += y_pred

        return pred_res

    def inference(self, feature):
        raise NotImplementedError

    def synthesis_by_vocoder(self, pred):
        audios_pred = synthesis(
            self.vocoder_cfg,
            self.checkpoint_dir_vocoder,
            len(pred),
            pred,
        )
        return audios_pred

    def __call__(self, utt):
        feature = self.build_test_utt_data(utt)
        start_time = time.time()
        with torch.no_grad():
            outputs = self.inference(feature)[0]
        time_used = time.time() - start_time
        rtf = time_used / (
            outputs.shape[1]
            * self.cfg.preprocess.hop_size
            / self.cfg.preprocess.sample_rate
        )
        print("Time used: {:.3f}, RTF: {:.4f}".format(time_used, rtf))
        self.avg_rtf.append(rtf)
        audios = outputs.cpu().squeeze().numpy().reshape(-1, 1)
        return audios

def base_parser():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--config", default="config.json", help="json files for configurations."
    )
    parser.add_argument("--use_ddp_inference", default=False)
    parser.add_argument("--n_workers", default=1, type=int)
    parser.add_argument("--local_rank", default=-1, type=int)
    parser.add_argument(
        "--batch_size", default=1, type=int, help="Batch size for inference"
    )
    parser.add_argument(
        "--num_workers",
        default=1,
        type=int,
        help="Worker number for inference dataloader",
    )
    parser.add_argument(
        "--checkpoint_dir",
        type=str,
        default=None,
        help="Checkpoint dir including model file and configuration",
    )
    parser.add_argument(
        "--checkpoint_file", help="checkpoint file", type=str, default=None
    )
    parser.add_argument(
        "--test_list", help="test utterance list for testing", type=str, default=None
    )
    parser.add_argument(
        "--checkpoint_dir_vocoder",
        help="Vocoder's checkpoint dir including model file and configuration",
        type=str,
        default=None,
    )
    parser.add_argument(
        "--output_dir",
        type=str,
        default=None,
        help="Output dir for saving generated results",
    )
    return parser

if __name__ == "__main__":
    parser = base_parser()
    args = parser.parse_args()
    cfg = load_config(args.config)

    # Build inference
    inference = BaseInference(cfg, args)
    inference()


import time
from datetime import datetime, timedelta

import schedule
from src.endpoints.util import get_deletion_cutoff
from src.processes.system_event_writer import system_event
from src.stateful_connectors.mongo_connector import delete_documents

def delete_old_documents():
    schedule.every(15).minutes.at(":40").do(_delete_old_documents)

    while True:
        schedule.run_pending()
        time.sleep(1)

def _delete_old_documents():
    print("Deleting old documents")

    dt = datetime.now()

    expired_candles_query = {
        "$or": [
            {"$and": [
                {"granularity": "M"},
                # 10 years
                {"time": {"$lte": get_deletion_cutoff(hours=87600)}}
            ]},
            {"$and": [
                {"granularity": "D"},
                {"time": {"$lte": get_deletion_cutoff(hours=35040)}}  # 4 years
            ]},
            {"$and": [
                {"granularity": "H12"},
                {"time": {"$lte": get_deletion_cutoff(hours=2880)}}  # 4 months
            ]},
            {"$and": [
                {"granularity": "H1"},
                {"time": {"$lte": get_deletion_cutoff(hours=960)}}  # 40 days
            ]},
            {"$and": [
                {"granularity": "M5"},
                {"time": {"$lte": get_deletion_cutoff(hours=72)}}  # 3 days
            ]}
        ]
    }

    delete_documents("candle", expired_candles_query)

    expired_system_actions = {
        "time": {"$lte": datetime.now() - timedelta(days=30)}}
    delete_documents("system_action", expired_system_actions)

    prices_cutoff = get_deletion_cutoff(hours=5)
    expired_prices = {"time": {"$lte": prices_cutoff}}
    delete_documents("price", expired_prices)

    expired_metrics = {"time": {"$lte": get_deletion_cutoff(hours=250)}} # approx 10 days
    delete_documents("metric", expired_metrics)

    system_event("data_analyst", "_delete_old_documents", dt, remarks={"start": dt,
                                                                       "end": datetime.now()})
    print("Done deleting documents")


import logging
import requests

from odoo import models, fields, api
from odoo.exceptions import ValidationError

_logger = logging.getLogger(__name__)

class CrmFacebookPage(models.Model):
    _name = 'crm.facebook.page'
    _description = 'Facebook Page'

    label = fields.Char(string='Page Label')
    # TODO: rename to id
    name = fields.Char(required=True, string='Page ID')
    access_token = fields.Char(required=True, string='Page Access Token')
    form_ids = fields.One2many('crm.facebook.form', 'page_id', string='Lead Forms')

    _sql_constraints = [
        ('name_unique', 'unique(name)', 'You cannot create a Page twice')
    ]

    @api.depends('label', 'name')
    def name_get(self):
        result = []
        for page in self:
            name = page.label if page.label else page.name
            result.append((page.id, name))
        return result

    def form_processing(self, r):
        if not r.get('data'):
            return
        for form in r['data']:
            if self.form_ids.filtered(
                    lambda f: f.facebook_form_id == form['id']):
                continue
            if form['status'] == 'ACTIVE':
                self.env['crm.facebook.form'].create({
                    'name': form['name'],
                    'facebook_form_id': form['id'],
                    'page_id': self.id}).get_fields()

        if r.get('paging') and r['paging'].get('next'):
            self.form_processing(requests.get(r['paging']['next']).json())
        return

    def get_forms(self):
        r = requests.get("https://graph.facebook.com/v7.0/" + self.name + "/leadgen_forms",
                         params={'access_token': self.access_token}).json()
        if r.get('error'):
            raise ValidationError(r['error']['message'])
        self.form_processing(r)

class CrmFacebookForm(models.Model):
    _name = 'crm.facebook.form'
    _description = 'Facebook Form Page'

    name = fields.Char(required=True)
    facebook_form_id = fields.Char(required=True, string='Form ID')
    access_token = fields.Char(required=True, related='page_id.access_token', string='Page Access Token')
    page_id = fields.Many2one('crm.facebook.page', readonly=True, ondelete='cascade', string='Facebook Page')
    mappings = fields.One2many('crm.facebook.form.field', 'form_id')
    team_id = fields.Many2one('crm.team', domain=['|', ('use_leads', '=', True), ('use_opportunities', '=', True)],
                              string="Sales Team")
    campaign_id = fields.Many2one('utm.campaign')
    source_id = fields.Many2one('utm.source')
    medium_id = fields.Many2one('utm.medium')
    date_retrieval = fields.Datetime(string='Fetch Leads After')

    def get_fields(self):
        self.mappings.unlink()
        r = requests.get("https://graph.facebook.com/v7.0/" + self.facebook_form_id,
                         params={'access_token': self.access_token, 'fields': 'questions'}).json()
        if r.get('error'):
            raise ValidationError(r['error']['message'])
        if r.get('questions'):
            for question in r.get('questions'):
                self.env['crm.facebook.form.field'].create({
                    'form_id': self.id,
                    'name': question['label'],
                    'facebook_field': question['key'],
                    'odoo_field': self.env['crm.facebook.form.mapping'].search(
                        [('facebook_field', '=', question['key'])], limit=1) and self.env[
                                      'crm.facebook.form.mapping'].search([('facebook_field', '=', question['key'])],
                                                                          limit=1).odoo_field.id or ''
                })

    def action_guess_mapping(self):
        for rec in self:
            rec.mappings.action_guess_mapping()

class CrmFacebookFormField(models.Model):
    _name = 'crm.facebook.form.field'
    _description = 'Facebook form fields'

    form_id = fields.Many2one('crm.facebook.form', required=True, ondelete='cascade', string='Form')
    name = fields.Char()
    odoo_field = fields.Many2one('ir.model.fields',
                                 domain=[('model', '=', 'crm.lead'),
                                         ('store', '=', True),
                                         ('ttype', 'in', ('char',
                                                          'date',
                                                          'datetime',
                                                          'float',
                                                          'html',
                                                          'integer',
                                                          'monetary',
                                                          'many2one',
                                                          'selection',
                                                          'phone',
                                                          'text'))],
                                 ondelete='set null',
                                 required=False)
    facebook_field = fields.Char(required=True)

    def action_guess_mapping(self):
        for rec in self:
            mapping = self.env['crm.facebook.form.mapping'].search([('facebook_field', '=', rec.facebook_field)],
                                                                   limit=1)
            if mapping:
                rec.odoo_field = mapping.odoo_field

    _sql_constraints = [
        ('field_unique', 'unique(form_id, odoo_field, facebook_field)', 'Mapping must be unique per form')
    ]

class CrmFacebookFormMapping(models.Model):
    _name = 'crm.facebook.form.mapping'
    _description = 'Default field mapping for new forms'

    odoo_field = fields.Many2one('ir.model.fields',
                                 domain=[('model', '=', 'crm.lead'),
                                         ('store', '=', True),
                                         ('ttype', 'in', ('char',
                                                          'date',
                                                          'datetime',
                                                          'float',
                                                          'html',
                                                          'integer',
                                                          'monetary',
                                                          'many2one',
                                                          'selection',
                                                          'phone',
                                                          'text'))],
                                 ondelete='cascade',
                                 required=True)
    facebook_field = fields.Char(required=True)

    _sql_constraints = [
        ('map_unique', 'unique(odoo_field, facebook_field)', 'Default Mapping must be unique')
    ]


#
# @lc app=leetcode.cn id=133 lang=python3
#
# [133] 克隆图
#

# @lc code=start
"""
# Definition for a Node.
class Node:
    def __init__(self, val = 0, neighbors = None):
        self.val = val
        self.neighbors = neighbors if neighbors is not None else []
"""

from lib2to3.pytree import Node
from logging.config import valid_ident

class Solution:
    def cloneGraph(self, node: 'Node') -> 'Node':
        if not node:
            return None
        # 1. 查找所有节点
        nodes = self.find_nodes_by_bfs(node)
        # 2. 复制节点的val
        # 返回 mapping[node] = new_node (new_node只有val)
        mapping = self.copy_node(nodes)
        # 3. 复制边
        self.copy_edges(nodes, mapping)
        return mapping[node]

    def find_nodes_by_bfs(self, node):
        queue = collections.deque([node])
        visited = set([node])
        while queue:
            curr_node = queue.popleft()
            for neighbor in curr_node.neighbors:
                if neighbor in visited:
                    continue
                queue.append(neighbor)
                visited.add(neighbor)
        return list(visited)

    def copy_node(self, nodes):
        mapping = dict()
        for node in nodes:
            mapping[node] = Node(node.val)
        return mapping

    def copy_edges(self, nodes, mapping):
        for node in nodes:
            # 取出对应新的节点
            new_node = mapping[node]
            for neighbor in node.neighbors:
                # 取出节点的邻接点
                new_neighbor = mapping[neighbor]
                # 添加新节点的邻接点
                new_node.neighbors.append(new_neighbor)

# @lc code=end


import bs4
import csv
from dataclasses import dataclass, astuple
import requests

COUNTRIES_FILE = 'countries.csv'
RAW_DATA_SOURCE = 'https://countrycode.org/'

@dataclass
class Country:
    name: str
    country_code: str
    iso_code: str
    iso_code_long: str
    population: str = 'Unknown'
    area: str = 'Unknown'
    gdp: str = 'Unknown'
    scholar_status: str = 'Unknown'

    def __post_init__(self):
        if not self.name or not self.country_code or not self.iso_code or not self.iso_code_long:
            raise ValueError('Missing param for country.')

    def csv_items(self):
        return astuple(self)

    @property
    def scholar_url(self):
        return f'https://scholar.google.com.{self.iso_code}'

class Countries:
    def __init__(self):
        self.country_list = []

    def add_country(self, country):
        self.country_list.append(country)

    def save(self):
        with open(COUNTRIES_FILE, 'w', newline='') as f:
            writer = csv.writer(f, delimiter=',', quotechar='"')
            writer.writerow(
                ['name', 'country_code', 'iso_code', 'iso_code_long', 'population', 'area', 'gdp', 'scholar_status'])
            for country in self.country_list:
                writer.writerow(country.csv_items())

    @staticmethod
    def load():
        countries = Countries()
        with open(COUNTRIES_FILE, newline='') as f:
            f.readline()  # Eat title row
            for row in csv.reader(f, delimiter=',', quotechar='"'):
                countries.add_country(Country(*row))
        return countries

    @staticmethod
    def load_from_html(file_name):
        countries = Countries()
        with open(file_name) as f:
            rows = bs4.BeautifulSoup(f.read(), 'html.parser').table.tbody.find_all('tr')
            for tr in rows:
                name, country_code, iso_code, population, area, gdp = tr.find_all('td')
                iso_code, iso_code_long = iso_code.text.split(' / ')
                countries.add_country(Country(
                    name=name.text,
                    country_code=country_code.text,
                    iso_code=iso_code.lower(),
                    iso_code_long=iso_code_long.lower(),
                    population=population.text.replace(',', ''),
                    area=area.text.replace(',', ''),
                    gdp=gdp.text.replace(',', '')
                ))
        return countries

    def reset_google_scholar_status(self, skip_unavailable=True):
        for country in self.country_list:
            if skip_unavailable and country.scholar_status == 'Unavailable': continue
            country.scholar_status = 'Unknown'

    def test_google_scholar(self, update_status=False, skip_unavailable=True, skip_known=False, verbose=False):
        try:
            for country in self.country_list:
                prefix = f'{country.name:40s} {country.scholar_url}'
                if (skip_known and country.scholar_status != 'Unknown') or \
                        (skip_unavailable and country.scholar_status == 'Unavailable'):
                    if verbose: print(f'{prefix}    skipped')
                    continue
                try:
                    r = requests.get(url=country.scholar_url)
                    status = str(r.status_code)
                except Exception:
                    status = 'Unavailable'
                print(f'{prefix}    {status}')
                if update_status: country.scholar_status = status
        except KeyboardInterrupt:
            print('Interrupted.')
        finally:
            if update_status:
                self.save()

    def show_available_scholar(self, only_status_200=True):
        for country in self.country_list:
            if country.scholar_status == 'Unavailable' or \
                    (only_status_200 and country.scholar_status != '200'):
                continue
            print(f'{country.name:40s} {country.scholar_url}    {country.scholar_status}')

    def __str__(self):
        return '\n'.join(map(str, self.country_list))

    def __getitem__(self, idx):
        return self.country_list[idx]

def run():
    countries = Countries.load()
    countries.test_google_scholar(update_status=False, skip_known=False)
    # countries.show_available_scholar()

if __name__ == '__main__':
    run()
    # countries = Countries.load()
    # for c in countries:
    #     print(c)


import random
import os
import json
import librosa
from tqdm import tqdm
from glob import glob
from collections import defaultdict

from utils.util import has_existed
from preprocessors import GOLDEN_TEST_SAMPLES

def get_test_songs():
    golden_samples = GOLDEN_TEST_SAMPLES["opensinger"]
    # every item is a tuple (singer, song)
    golden_songs = [s.split("_")[:3] for s in golden_samples]
    # singer_song, eg: Female1#Almost_lover_Amateur
    return golden_songs

def opensinger_statistics(data_dir):
    singers = []
    songs = []
    singer2songs = defaultdict(lambda: defaultdict(list))

    gender_infos = glob(data_dir + "/*")

    for gender_info in gender_infos:
        gender_info_split = gender_info.split("/")[-1][:-3]

        singer_and_song_infos = glob(gender_info + "/*")

        for singer_and_song_info in singer_and_song_infos:
            singer_and_song_info_split = singer_and_song_info.split("/")[-1].split("_")
            singer_id, song = (
                singer_and_song_info_split[0],
                singer_and_song_info_split[1],
            )
            singer = gender_info_split + "_" + singer_id
            singers.append(singer)
            songs.append(song)

            utts = glob(singer_and_song_info + "/*.wav")

            for utt in utts:
                uid = utt.split("/")[-1].split("_")[-1].split(".")[0]
                singer2songs[singer][song].append(uid)

    unique_singers = list(set(singers))
    unique_songs = list(set(songs))
    unique_singers.sort()
    unique_songs.sort()

    print(
        "opensinger: {} singers, {} songs ({} unique songs)".format(
            len(unique_singers), len(songs), len(unique_songs)
        )
    )
    print("Singers: \n{}".format("\t".join(unique_singers)))
    return singer2songs, unique_singers

def main(output_path, dataset_path):
    print("-" * 10)
    print("Preparing test samples for opensinger...\n")

    save_dir = os.path.join(output_path, "opensinger")
    os.makedirs(save_dir, exist_ok=True)
    train_output_file = os.path.join(save_dir, "train.json")
    test_output_file = os.path.join(save_dir, "test.json")
    singer_dict_file = os.path.join(save_dir, "singers.json")
    utt2singer_file = os.path.join(save_dir, "utt2singer")
    if (
        has_existed(train_output_file)
        and has_existed(test_output_file)
        and has_existed(singer_dict_file)
        and has_existed(utt2singer_file)
    ):
        return
    utt2singer = open(utt2singer_file, "w")

    # Load
    opensinger_path = dataset_path

    singer2songs, unique_singers = opensinger_statistics(opensinger_path)
    test_songs = get_test_songs()

    # We select songs of standard samples as test songs
    train = []
    test = []

    train_index_count = 0
    test_index_count = 0

    train_total_duration = 0
    test_total_duration = 0

    for i, (singer, songs) in enumerate(singer2songs.items()):
        song_names = list(songs.keys())

        for chosen_song in tqdm(
            song_names, desc="Singer {}/{}".format(i, len(singer2songs))
        ):
            for chosen_uid in songs[chosen_song]:
                res = {
                    "Dataset": "opensinger",
                    "Singer": singer,
                    "Song": chosen_song,
                    "Uid": "{}_{}_{}".format(singer, chosen_song, chosen_uid),
                }
                res["Path"] = "{}Raw/{}_{}/{}_{}_{}.wav".format(
                    singer.split("_")[0],
                    singer.split("_")[1],
                    chosen_song,
                    singer.split("_")[1],
                    chosen_song,
                    chosen_uid,
                )
                res["Path"] = os.path.join(opensinger_path, res["Path"])
                assert os.path.exists(res["Path"])

                duration = librosa.get_duration(filename=res["Path"])
                res["Duration"] = duration

                if duration > 30:
                    print(
                        "Wav file: {}, the duration = {:.2f}s > 30s, which has been abandoned.".format(
                            res["Path"], duration
                        )
                    )
                    continue

                if (
                    [singer.split("_")[0], singer.split("_")[1], chosen_song]
                ) in test_songs:
                    res["index"] = test_index_count
                    test_total_duration += duration
                    test.append(res)
                    test_index_count += 1
                else:
                    res["index"] = train_index_count
                    train_total_duration += duration
                    train.append(res)
                    train_index_count += 1

                utt2singer.write("{}\t{}\n".format(res["Uid"], res["Singer"]))

    print("#Train = {}, #Test = {}".format(len(train), len(test)))
    print(
        "#Train hours= {}, #Test hours= {}".format(
            train_total_duration / 3600, test_total_duration / 3600
        )
    )

    # Save train.json and test.json
    with open(train_output_file, "w") as f:
        json.dump(train, f, indent=4, ensure_ascii=False)
    with open(test_output_file, "w") as f:
        json.dump(test, f, indent=4, ensure_ascii=False)

    # Save singers.json
    singer_lut = {name: i for i, name in enumerate(unique_singers)}
    with open(singer_dict_file, "w") as f:
        json.dump(singer_lut, f, indent=4, ensure_ascii=False)


import random
import sys
from mathutils import Matrix, Vector
import time
import numpy as np
import math
from bpy.types import Menu
import bmesh
import bpy
from bl_ui.utils import PresetPanel
from bpy.types import Panel
from rna_prop_ui import PropertyPanel
from bpy_extras.node_utils import find_node_input
from bpy_extras import (asset_utils,)

# ------------------------------------------------------------------------------
# LIGHT PANEL

class LIGHT_PT_Panel (bpy.types.Panel):
    bl_label = "AB Light"
    bl_idname = "LIGHT_PT_Panel"
    bl_space_type = "VIEW_3D"
    bl_region_type = "UI"
    bl_category = "Arc Blend"
    bl_options = {'DEFAULT_CLOSED'}

    def draw(self, context):
        layout = self.layout
        ob = context.object
        scene = context.scene
        cam = context.scene.camera
        col = layout.column()
        row = layout.row()
        col.label(text="Light Panel :" , icon="LIGHT")
        col.operator("object.button_light_point", text="Add Light to Scene", icon="OUTLINER_OB_LIGHT")
        # draw template
        col1 = row.column()
        col2 = row.column()
        template = col1
        template.template_list("LIGHT_UL_List", "light_list", scene, "light_list",
                                   scene, "light_list_index", rows=3)
        template.scale_y = 1.1
        # draw side bar
        col.separator(factor=0)

        #
        add = col2.column(align=True)
        AB_ADD = add.operator(
            "object.button_light_panel_add_light", icon='ADD', text="")

        rem = col2.column(align=True)
        AB_REMOVE = rem.operator(
            "object.button_light_panel_remove_light", icon='REMOVE', text="")

        col.column_flow(columns=3, align=True)

# ------------------------------------------------------------------------------
# LIGHT LIST

class LIGHT_UL_List(bpy.types.UIList):
    """UI Light List"""

    def draw_item(self, context, layout, data, item, icon, active_data,
                  active_propname, index):

        scene = context.scene
        row = layout.row(align=True)
        sub = row.row(align=True)
        sub.scale_x = 2
        sub.prop(item, "light_item", text='', icon="OUTLINER_OB_LIGHT")
        sub = row.row(align=True)
        sub.scale_x = 1.1
        sub.enabled = bool(item.light_item)
        sub.prop(item, "light_display", text='',
                 icon='RESTRICT_VIEW_OFF' if item.light_display else 'RESTRICT_VIEW_ON')
        sub = row.row(align=True)
        sub.scale_x = 0.3
        sub.prop(item, "name", text="", icon="LAYER_ACTIVE", emboss=False)
        sub.enabled = False

# ------------------------------------------------------------------------------
# ADD LIGHT TO LIST

class light_panel_add_light(bpy.types.Operator):
    """Add a new item to the list"""

    bl_label = ""
    bl_idname = "object.button_light_panel_add_light"

    def execute(self, context):
        try:
            light_list_scn = []
            for i in bpy.context.scene.objects[:]:
                if i.type == "LIGHT":
                    light_list_scn.append(i)
            for i in light_list_scn:
                scene = bpy.context.scene
                light = i
                index = scene.light_list_index
                item = scene.light_list.add()
                item.name = light.name
                item.light_ui_index = len(scene.light_list)
                scene.light_list_index = len(scene.light_list) - 1

                # Break after adding one item to the list
                break
        except (AttributeError, KeyError, UnboundLocalError):
            pass

        return {'FINISHED'}

# ------------------------------------------------------------------------------
# DELETE OBJECT FROM LIST LIGHT

class light_panel_remove_light(bpy.types.Operator):
    """Remove items from the list"""

    bl_label = ""
    bl_idname = "object.button_light_panel_remove_light"

    def execute(self, context):
        try:
            light_list_scn = []
            for i in bpy.context.scene.objects[:]:
                if i.type == "LIGHT":
                    light_list_scn.append(i)
            for i in light_list_scn:
                scene = bpy.context.scene
                light = i
                index = scene.light_list_index
                scene.light_list_index -= 1
                scene.light_list.remove(index)
                # Break after adding one item to the list
                break
        except (AttributeError, KeyError, UnboundLocalError):
            pass

        return {'FINISHED'}

# ------------------------------------------------------------------------------
# DEF LIGHT
def light_return_one_time(scene, active_idx, prop_api):
    """Returning once per elements"""
    AB_list = scene.light_list

    for i in AB_list:
        if i.light_ui_index != active_idx:
            exec(f"i.{prop_api}= False")
    return None

def light_display_upd(self, context):

    if self.light_display == True:
        light_return_one_time(self.id_data, self.light_ui_index, "light_display")

        for i in bpy.data.scenes[bpy.context.scene.name].light_list[:]:
               if i.light_display== True:
                   bpy.context.active_object.select_set(False)
                   bpy.context.view_layer.objects.active = bpy.data.objects[i.name]
                   bpy.context.object.select_set(True)

    else:
        pass

# ------------------------------------------------------------------------------
# LIGHT PROPERTY GROUP

class light_panel_list_item(bpy.types.PropertyGroup):
    id: bpy.props.IntProperty()
    light: bpy.props.PointerProperty(name="Light",type=bpy.types.Light)

    # Name of the items in the list
    name: bpy.props.StringProperty(description="Light Name")
    # Random props in the lists
    light_ui_index: bpy.props.IntProperty(description='UI List Index')
    light_item: bpy.props.PointerProperty(type=bpy.types.Light, description='Light Name')
    light_display: bpy.props.BoolProperty(default=False, description="Display in Viewport", update=light_display_upd)

    def copy(self):
        self.light = self.id_data.copy()
        self.name = self.light.name
        return self.light

    def add(self, light):
        self.light = light
        self.name = light.name
        return self.light

# ------------------------------------------------------------------------------
# LIGHT BUTTON PANEL

class DataButtonsPanel:
    bl_space_type = 'PROPERTIES'
    bl_region_type = 'WINDOW'
    bl_context = "data"

    @classmethod
    def poll(cls, context):
        active = bpy.context.active_object
        try:
            if active.type == "LIGHT":
                engine = context.engine
                return active.data and (engine in cls.COMPAT_ENGINES)
        except (AttributeError,KeyError):
            pass
# ------------------------------------------------------------------------------
# LIGHT BUTTON PANEL

class LIGHT_PT_List_Context_Light(DataButtonsPanel, Panel):
    bl_label = ""
    bl_options = {'HIDE_HEADER'}
    COMPAT_ENGINES = {'BLENDER_RENDER', 'BLENDER_EEVEE', 'BLENDER_WORKBENCH'}
    bl_space_type = "VIEW_3D"
    bl_region_type = "UI"
    bl_category = "Arc Blend"
    bl_parent_id = "LIGHT_PT_Panel"

    def draw(self, context):
        layout = self.layout
        ob = context.object
        space = context.space_data

        if ob:
            layout.template_ID(ob, "data")
        elif light:
            layout.template_ID(space, "pin_id")

# ------------------------------------------------------------------------------
# LIGHT PANEL LIST Light x

class LIGHT_PT_List_EEVEE_Light(DataButtonsPanel, Panel):
    bl_label = "Light"
    bl_options = {'DEFAULT_CLOSED'}
    bl_space_type = "VIEW_3D"
    bl_region_type = "UI"
    bl_category = "Arc Blend"
    bl_parent_id = "LIGHT_PT_Panel"
    COMPAT_ENGINES = {'BLENDER_EEVEE'}

    def draw(self, context):
        active = bpy.context.active_object
        try:
            if active.type == "LIGHT":
                context = active.data
                layout = self.layout
                light = context
                # Compact layout for node editor.
                layout.use_property_split = False
                layout.row().prop(light, "type",expand=True)
                col = layout.column()
                col.prop(light, "color")
                col.prop(light, "energy")
                col.separator()
                col.prop(light, "diffuse_factor", text="Diffuse")
                col.prop(light, "specular_factor", text="Specular")
                col.prop(light, "volume_factor", text="Volume")
                col.separator()
                if light.type in {'POINT', 'SPOT'}:
                    col.prop(light, "shadow_soft_size", text="Radius")
                elif light.type == 'SUN':
                    col.prop(light, "angle")
                elif light.type == 'AREA':
                    col.prop(light, "shape")
                    sub = col.column(align=True)
                    if light.shape in {'SQUARE', 'DISK'}:
                        sub.prop(light, "size")
                    elif light.shape in {'RECTANGLE', 'ELLIPSE'}:
                        sub.prop(light, "size", text="Size X")
                        sub.prop(light, "size_y", text="Y")
        except (AttributeError,KeyError):
            pass
# ------------------------------------------------------------------------------
# LIGHT PANEL distance
class LIGHT_PT_List_EEVEE_Light_Distance(DataButtonsPanel, Panel):
    bl_label = "Custom Distance"
    bl_parent_id = "LIGHT_PT_List_EEVEE_Light"
    bl_space_type = "VIEW_3D"
    bl_region_type = "UI"
    bl_category = "Arc Blend"
    bl_options = {'DEFAULT_CLOSED'}
    COMPAT_ENGINES = {'BLENDER_EEVEE'}

    @classmethod
    def poll(cls, context):
        active = bpy.context.active_object
        try:
            if active.type == "LIGHT":
                context1 = active.data
                light = context1
                engine = context.engine
                return (light and light.type != 'SUN') and (engine in cls.COMPAT_ENGINES)
        except (AttributeError,KeyError):
            pass

    def draw_header(self, context):
        active = bpy.context.active_object
        try:
            for i in bpy.context.scene.objects[:]:
                if i.type == "LIGHT":
                    context = i.data
                    light = context
                    layout = self.layout
                    layout.prop(light, "use_custom_distance", text="")
        except (AttributeError,KeyError):
            pass
    def draw(self, context):
        active = bpy.context.active_object
        try:
            if active.type == "LIGHT":
                context = active.data
                layout = self.layout
                light = context
                layout.active = light.use_custom_distance
                layout.use_property_split = True

                layout.prop(light, "cutoff_distance", text="Distance")
        except (AttributeError,KeyError):
            pass
# ------------------------------------------------------------------------------
# LIGHT PANEL shadow

class LIGHT_PT_List_EEVEE_Shadow(DataButtonsPanel, Panel):
    bl_label = "Shadow"
    bl_space_type = "VIEW_3D"
    bl_region_type = "UI"
    bl_category = "Arc Blend"
    bl_parent_id = "LIGHT_PT_Panel"
    bl_options = {'DEFAULT_CLOSED'}
    COMPAT_ENGINES = {'BLENDER_EEVEE'}

    @classmethod
    def poll(cls, context):
        active = bpy.context.active_object
        try:
            if active.type == "LIGHT":
                context1 = active.data
                light = context1
                engine = context.engine
                return (
                    (light and light.type in {'POINT', 'SUN', 'SPOT', 'AREA'}) and
                    (engine in cls.COMPAT_ENGINES)
                )
        except (AttributeError,KeyError):
            pass
    def draw_header(self, context):
        active = bpy.context.active_object
        try:
            if active.type == "LIGHT":
                context = active.data
                light = context
                self.layout.prop(light, "use_shadow", text="")
        except (AttributeError,KeyError):
            pass

    def draw(self, context):
        active = bpy.context.active_object
        try:
            if active.type == "LIGHT":
                context = active.data
                layout = self.layout
                layout.use_property_split = True
                light = context
                layout.active = light.use_shadow
                col = layout.column()
                sub = col.column(align=True)
                if light.type != 'SUN':
                    sub.prop(light, "shadow_buffer_clip_start", text="Clip Start")
                col.prop(light, "shadow_buffer_bias", text="Bias")
        except (AttributeError,KeyError):
            pass
# ------------------------------------------------------------------------------
# LIGHT PANEL shadow map
class LIGHT_PT_List_EEVEE_Shadow_Cascaded_Shadow_Map(DataButtonsPanel, Panel):
    bl_label = "Cascaded Shadow Map"
    bl_parent_id = "LIGHT_PT_List_EEVEE_Shadow"
    bl_space_type = "VIEW_3D"
    bl_region_type = "UI"
    bl_category = "Arc Blend"
    bl_options = {'DEFAULT_CLOSED'}
    COMPAT_ENGINES = {'BLENDER_EEVEE'}

    @classmethod
    def poll(cls, context):
        active = bpy.context.active_object
        try:
            if active.type == "LIGHT":
                context1 = active.data
                light = context1
                engine = context.engine
                return (light and light.type == 'SUN') and (engine in cls.COMPAT_ENGINES)
        except (AttributeError,KeyError):
            pass
    def draw(self, context):
        active = bpy.context.active_object
        try:
            if active.type == "LIGHT":
                context = active.data
                layout = self.layout
                light = context
                layout.use_property_split = True

                col = layout.column()

                col.prop(light, "shadow_cascade_count", text="Count")
                col.prop(light, "shadow_cascade_fade", text="Fade")

                col.prop(light, "shadow_cascade_max_distance", text="Max Distance")
                col.prop(light, "shadow_cascade_exponent", text="Distribution")
        except (AttributeError,KeyError):
            pass
# ------------------------------------------------------------------------------
# LIGHT PANEL shadow contact
class LIGHT_PT_List_EEVEE_Shadow_Contact(DataButtonsPanel, Panel):
    bl_label = "Contact Shadows"
    bl_parent_id = "LIGHT_PT_List_EEVEE_Shadow"
    bl_space_type = "VIEW_3D"
    bl_region_type = "UI"
    bl_category = "Arc Blend"
    COMPAT_ENGINES = {'BLENDER_EEVEE'}

    @classmethod
    def poll(cls, context):
        active = bpy.context.active_object
        try:
            if active.type == "LIGHT":
                context1 = active.data
                light = context1
                engine = context.engine
                return (
                    (light and light.type in {'POINT', 'SUN', 'SPOT', 'AREA'}) and
                    (engine in cls.COMPAT_ENGINES)
                )
        except (AttributeError,KeyError):
            pass

    def draw_header(self, context):
        active = bpy.context.active_object
        try:
            if active.type == "LIGHT":
                context = active.data
                light = context
                layout = self.layout
                layout.active = light.use_shadow
                layout.prop(light, "use_contact_shadow", text="")
        except (AttributeError,KeyError):
            pass

    def draw(self, context):
        active = bpy.context.active_object
        try:
            if active.type == "LIGHT":
                context = active.data
                layout = self.layout
                light = context
                layout.use_property_split = True

                col = layout.column()
                col.active = light.use_shadow and light.use_contact_shadow

                col.prop(light, "contact_shadow_distance", text="Distance")
                col.prop(light, "contact_shadow_bias", text="Bias")
                col.prop(light, "contact_shadow_thickness", text="Thickness")
        except (AttributeError,KeyError):
            pass

# ------------------------------------------------------------------------------
# LIGHT PANEL area
class LIGHT_PT_List_Area(DataButtonsPanel, Panel):
    bl_label = "Area Shape"
    COMPAT_ENGINES = {'BLENDER_RENDER', 'BLENDER_WORKBENCH'}
    bl_space_type = "VIEW_3D"
    bl_region_type = "UI"
    bl_category = "Arc Blend"
    bl_parent_id = "LIGHT_PT_Panel"

    @classmethod
    def poll(cls, context):
        active = bpy.context.active_object
        try:
            if active.type == "LIGHT":
                context1 = active.data
                light = context1
                engine = context.engine
                return (light and light.type == 'AREA') and (engine in cls.COMPAT_ENGINES)
        except AttributeError:
            pass
    def draw(self, context):
        active = bpy.context.active_object
        try:
            if active.type == "LIGHT":
                context = active.data
                layout = self.layout
                light = context
                col = layout.column()
                col.row().prop(light, "shape", expand=True)
                sub = col.row(align=True)
                if light.shape in {'SQUARE', 'DISK'}:
                    sub.prop(light, "size")
                elif light.shape in {'RECTANGLE', 'ELLIPSE'}:
                    sub.prop(light, "size", text="Size X")
                    sub.prop(light, "size_y", text="Size Y")
        except AttributeError:
            pass
# ------------------------------------------------------------------------------
# LIGHT PANEL spot
class LIGHT_PT_List_Spot(DataButtonsPanel, Panel):
    bl_label = "Spot Shape"
    bl_parent_id = "LIGHT_PT_List_EEVEE_Light"
    COMPAT_ENGINES = {'BLENDER_RENDER', 'BLENDER_EEVEE', 'BLENDER_WORKBENCH'}
    bl_space_type = "VIEW_3D"
    bl_region_type = "UI"
    bl_category = "Arc Blend"

    @classmethod
    def poll(cls, context):
        active = bpy.context.active_object
        try:
            if active.type == "LIGHT":
                context1 = active.data
                light = context1
                engine = context.engine
                return (light and light.type == 'SPOT') and (engine in cls.COMPAT_ENGINES)
        except AttributeError:
            pass
    def draw(self, context):
        active = bpy.context.active_object
        try:
            if active.type == "LIGHT":
                context = active.data
                layout = self.layout
                layout.use_property_split = True
                light = context
                col = layout.column()
                col.prop(light, "spot_size", text="Size")
                col.prop(light, "spot_blend", text="Blend", slider=True)
                col.prop(light, "show_cone")
        except AttributeError:
            pass
# ------------------------------------------------------------------------------
# LIGHT PANEL falloff curve
class LIGHT_PT_List_Falloff_Curve(DataButtonsPanel, Panel):
    bl_label = "Falloff Curve"
    bl_options = {'DEFAULT_CLOSED'}
    COMPAT_ENGINES = {'BLENDER_RENDER', 'BLENDER_EEVEE'}

    @classmethod
    def poll(cls, context):
        active = bpy.context.active_object
        try:
            if active.type == "LIGHT":
                context1 = active.data
                light = context1
                engine = context.engine

                return (
                    (light and light.type in {'POINT', 'SPOT'} and light.falloff_type == 'CUSTOM_CURVE') and
                    (engine in cls.COMPAT_ENGINES)
                )
        except AttributeError:
            pass
    def draw(self, context):
        active = bpy.context.active_object
        try:
            if active.type == "LIGHT":
                context = active.data
                light = context

                self.layout.template_curve_mapping(
                    light, "falloff_curve", use_negative_slope=True)
        except AttributeError:
            pass

# ------------------------------------------------------------------------------
# REGISTRATION AREA

def register():
    #Light
    bpy.utils.register_class(LIGHT_PT_Panel)
    bpy.utils.register_class(LIGHT_UL_List)
    bpy.utils.register_class(light_panel_add_light)
    bpy.utils.register_class(light_panel_remove_light)
    bpy.utils.register_class(light_panel_list_item)
    #Light Scene
    bpy.types.Scene.light_list = bpy.props.CollectionProperty(type=light_panel_list_item)
    bpy.types.Scene.light_list_index = bpy.props.IntProperty(name="Index for light_list", default=0)
    bpy.utils.register_class(LIGHT_PT_List_Context_Light)
    bpy.utils.register_class(LIGHT_PT_List_EEVEE_Light)
    bpy.utils.register_class(LIGHT_PT_List_EEVEE_Light_Distance)
    bpy.utils.register_class(LIGHT_PT_List_EEVEE_Shadow)
    bpy.utils.register_class(LIGHT_PT_List_EEVEE_Shadow_Cascaded_Shadow_Map)
    bpy.utils.register_class(LIGHT_PT_List_EEVEE_Shadow_Contact)
    bpy.utils.register_class(LIGHT_PT_List_Area)
    bpy.utils.register_class(LIGHT_PT_List_Spot)
    bpy.utils.register_class(LIGHT_PT_List_Falloff_Curve)

def unregister():
     #Light
    bpy.utils.unregister_class(LIGHT_PT_Panel)
    bpy.utils.unregister_class(LIGHT_UL_List)
    bpy.utils.unregister_class(light_panel_add_light)
    bpy.utils.unregister_class(light_panel_remove_light)
    bpy.utils.unregister_class(light_panel_list_item)
    #Light Scene
    del bpy.types.Scene.light_list
    del bpy.types.Scene.light_list_index
    bpy.utils.unregister_class(LIGHT_PT_List_Context_Light)
    bpy.utils.unregister_class(LIGHT_PT_List_EEVEE_Light)
    bpy.utils.unregister_class(LIGHT_PT_List_EEVEE_Light_Distance)
    bpy.utils.unregister_class(LIGHT_PT_List_EEVEE_Shadow)
    bpy.utils.unregister_class(LIGHT_PT_List_EEVEE_Shadow_Cascaded_Shadow_Map)
    bpy.utils.unregister_class(LIGHT_PT_List_EEVEE_Shadow_Contact)
    bpy.utils.unregister_class(LIGHT_PT_List_Area)
    bpy.utils.unregister_class(LIGHT_PT_List_Spot)
    bpy.utils.unregister_class(LIGHT_PT_List_Falloff_Curve)


import os
import sys
sys.path.append(os.getcwd())
import shutil
from argparse import ArgumentParser
import darwin.models.mimic_image as mimic_image
import darwin.models.mimic_static as mimic_static
import darwin.models.mimic_image_static as mimic_image_static
import darwin.models.mimic_timeseries as mimic_timeseries
import torch
import darwin.config as config

# Script used to perform an experiment

#Expected format of the data output to the experiments directory:
#Root directory - > experiment directory - > ModelName ->
#- A csv file for each run of the model containing its accuracy on validation data while training
#- A model_name-test.csv file containing accuracy on the test data for each run.

def t_or_f(arg):
    ua = str(arg).upper()
    if 'TRUE'.startswith(ua):
       return True
    elif 'FALSE'.startswith(ua):
       return False
    else:
       print("ERROR!!")

def main():
    parser = ArgumentParser(prog='experiment.py')
    parser.add_argument('-nr', '--numRuns', default='5',type=int)
    parser.add_argument('-ne', '--numEpochs', default='20',type=int)
    parser.add_argument('-bs', '--batchSize', default='3',type=int,help = "Batch size while training")
    parser.add_argument('-pat', '--patience', default='7',type=int,help = "Used with --earlyStop. Number of epochs to wait for an improvement for.")
    parser.add_argument('-estm', '--earlyStopMetric',default = 'acc', choices = ['valid', 'acc'],help = "What metric to use for early stopping.")
    parser.add_argument('-sf', '--shuffle',default = 'True', choices = [True, False],type=t_or_f,help = "Whether to use a pre-determined split or to randomly shuffle the split between each run.")
    parser.add_argument('-care', '--careful',default = 'True', choices = [True, False],type=t_or_f,help = "Whether to have the script ask you if you're sure if you want to \
overwrite the experiments folder. Turning this off helps if you want to run the tests via a bash or batch script")
    parser.add_argument('-m', '--model',required= True, choices = ['static', 'image', 'image_static', 'timeseries','static_timeseries', 'timeseries_image','static_timeseries_image'])
    parser.add_argument('-o', '--rootDir',required = True, help = "Directory where the experiments are saved to. Note that this path should be written with forward slashes.")
    parser.add_argument('-en', '--experimentName',required = True, help = "Name of the experiment. A folder with this name is created inside the root directory where all results will be output to.")
    parser.add_argument('-lr', '--learningRate',default = '0.001', help = "Learning rate of the model while training.",type=float)
    parser.add_argument('-dp', '--dropOutP',default = '0', help = "DOES NOT WORK. Weird bug, if anyone wants to help with that, that'd be great. Dropout percentage of the static model while training.",type=float)
    parser.add_argument('-kf', '--kFold',default = '0', help = "Whether to perform kfold cross validation. Number of folds to do. If < 2, then kfold cross validation won't be performed",type=int)
    parser.add_argument('-opt', '--optimizer',default = 'RMSprop', choices = ['adam', 'RMSprop'], help = "Optimizer to use while training")
    parser.add_argument('-estp', '--earlyStop',default = 'True', choices = [True, False],type=t_or_f,help = "Whether to stop after 7 epochs where validation accuracy did not improve.")
    parser.add_argument('-aug', '--augmentImage',default = 'False', choices = [True, False],type=t_or_f,help = "Whether to augment the images whenver they are retreived from the dataloader.\
This should increase the image model's ability to generalize")
    parser.add_argument('-save', '--saveModels',default = 'False', choices = [True, False],type=t_or_f,help = "Whether to save the model with best accuracy on validation for every run. This will also save the test dataloader used\
by the model. Note this will save as many models as runs.")

    args = parser.parse_args()

    rootExperimentDir = args.rootDir + "/{}".format(args.experimentName)
    modelResultsDir = rootExperimentDir + "/{}".format(args.model)

    if args.optimizer == 'RMSprop':
        optimizer = torch.optim.RMSprop
    elif args.optimizer == 'adam':
        optimizer = torch.optim.Adam

    print("\n\nPerforming experiment {} on the {} model for {} epochs and {} runs. \nThe results will be stored in the: '{}' directory \n\n".format(args.experimentName,args.model,str(args.numEpochs),str(args.numRuns),modelResultsDir))

    print("* Early stop = {}\n".format(str(args.earlyStop)))

    print("* Patience : {}\n".format(args.patience))

    print("* Early stop metric : {}\n".format(args.earlyStopMetric))

    if args.shuffle and args.kFold < 2:
        print("* The valiation/training/testing split will be randomly shuffled for each run. The shuffle will be stratified.\n")
    elif args.kFold < 2:
        print("* The valiation/training/testing split will be the same for every run.\n")
    elif args.kFold >=2:
        print("* Every run will be a {}-fold cross validation\n".format(args.kFold))

    print("* Learning rate: {}\n".format(str(args.learningRate)))

    if (args.model == 'static' or args.model == 'image_static'):
        print("* Dropout percentage of the static model is: {}\n".format(args.dropOutP))

    print("* Optimizer used: {}\n".format(args.optimizer))

    print("* Batch size: {}\n".format(args.batchSize))

    print("* Augment images: {}\n".format(args.augmentImage))

    print("* Save models: {}\n \n".format(args.saveModels))

    if not os.path.exists(args.rootDir):
        os.makedirs(args.rootDir)
    if not os.path.exists(rootExperimentDir):
        os.makedirs(rootExperimentDir)
    if not os.path.exists(modelResultsDir):
        os.makedirs(modelResultsDir)
    else:
        #If the directory containing results exists, delete it.
        if (args.careful == False):
            shutil.rmtree(modelResultsDir)
            os.makedirs(modelResultsDir)
        else:
            #A little safeguard in place just in case the user accidentally somehow inputted the wrong outputdir. And the results directory ends up being a valuable directory in their system.
            answer = 'asdgsd'
            while (not (answer == 'y' or answer == 'n')):
                answer = input("The script is about to delete the {} folder and recursively delete everything under it in order to replace it with the experiment results.\n\
Are you sure you want to continue?: (y/n)".format(modelResultsDir))
            if (answer == 'y'):
                shutil.rmtree(modelResultsDir)
                os.makedirs(modelResultsDir)
            else:
                print("You've terminated the script")
                return -1

    # Print out the arguments used to the script in the experiment folder
    with open(modelResultsDir + '/arguments.txt', 'w') as f:
        for arg in sys.argv:
            f.write(arg + ' ')
        f.write("\n Impk path : {}\n".format(config.dataPath))

    # Run the model

    if (args.model == "static"):
        mimic_static.runModel(args.numRuns,modelResultsDir,args.numEpochs,args.shuffle,args.learningRate,args.dropOutP != 0.0,args.dropOutP,optimizer=optimizer,
        earlyStop=args.earlyStop,kfold=args.kFold,batch_size=args.batchSize,save_models=args.saveModels,patience=args.patience,early_stop_metric=args.earlyStopMetric)
    elif (args.model == "image"):
        mimic_image.runModel(args.numRuns,modelResultsDir,args.numEpochs,args.shuffle,args.learningRate,optimizer=optimizer,
        earlyStop=args.earlyStop,kfold=args.kFold,batch_size=args.batchSize,augmentImages=args.augmentImage,save_models=args.saveModels,patience=args.patience,early_stop_metric=args.earlyStopMetric)
    elif (args.model == "image_static"):
        mimic_image_static.runModel(args.numRuns,modelResultsDir,args.numEpochs,args.shuffle,args.learningRate,args.dropOutP != 0.0,args.dropOutP,optimizer=optimizer,
        earlyStop=args.earlyStop,kfold=args.kFold,batch_size=args.batchSize,augmentImages=args.augmentImage,save_models=args.saveModels,patience=args.patience,early_stop_metric=args.earlyStopMetric)
    elif (args.model == "timeseries"):
        mimic_timeseries.runModel(args.numRuns,modelResultsDir,args.numEpochs,args.shuffle,args.learningRate,optimizer=optimizer,
        earlyStop=args.earlyStop,kfold=args.kFold,batch_size=args.batchSize,save_models=args.saveModels,patience=args.patience,early_stop_metric=args.earlyStopMetric)
    elif (args.model == "static_timeseries"):
        mimic_static.runModel(args.numRuns,modelResultsDir,args.numEpochs,args.shuffle,args.learningRate,args.dropOutP != 0.0,args.dropOutP,optimizer=optimizer,
        earlyStop=args.earlyStop,kfold=args.kFold,batch_size=args.batchSize,save_models=args.saveModels,patience=args.patience,early_stop_metric=args.earlyStopMetric)
    elif (args.model == "timeseries_image"):
        mimic_image.runModel(args.numRuns,modelResultsDir,args.numEpochs,args.shuffle,args.learningRate,optimizer=optimizer,
        earlyStop=args.earlyStop,kfold=args.kFold,batch_size=args.batchSize,augmentImages=args.augmentImage,save_models=args.saveModels,patience=args.patience,early_stop_metric=args.earlyStopMetric)
    elif (args.model == "static_timeseries_image"):
        mimic_image_static.runModel(args.numRuns,modelResultsDir,args.numEpochs,args.shuffle,args.learningRate,args.dropOutP != 0.0,args.dropOutP,optimizer=optimizer,
        earlyStop=args.earlyStop,kfold=args.kFold,batch_size=args.batchSize,augmentImages=args.augmentImage,save_models=args.saveModels,patience=args.patience,early_stop_metric=args.earlyStopMetric)

if __name__ == '__main__':
    main()


#Please copy the completed function from above into this active code window. Now write a function called get_movie_rating.
#It takes an OMDB dictionary result for one movie and extracts the Rotten Tomatoes rating as an integer.
#For example, if given the OMDB dictionary for “Black Panther”, it would return 97. If there is no Rotten Tomatoes rating, return 0.

# some invocations that we use in the automated tests; uncomment these if you are getting errors and want better error messages
# get_movie_rating(get_movie_data("Deadpool 2"))
import requests_with_caching
import json

def get_movie_data(title):
    endpoint = 'http://www.omdbapi.com/'
    param = {}
    param['t'] = title
    param['r'] = 'json'
    this_page_cache = requests_with_caching.get(endpoint, params=param)
    return json.loads(this_page_cache.text)
print(get_movie_data("Black Panther")['Ratings'][1])
def get_movie_rating(dic):
    ranking = dic['Ratings']
    for dic_item in ranking:
        if dic_item['Source'] == 'Rotten Tomatoes':
            return int(dic_item['Value'][:-1])
    return 0

get_movie_rating(get_movie_data("Deadpool 2"))

Python 3 Programming/Course 3- Data Collection and Processing with Python/Assessment 3 - Project OMDB and TasteDive Mashup/Assessment 3.5.py


import pytest
from unittest.mock import patch

import pandas as pd
import pandas.util.testing as pdt
from flask import Flask

from pyaddepar.flask_addepar import addepar
from pyaddepar.addeparrequest import AddeparRequest, PortfolioType, AttrDict

@pytest.fixture
def addepar_request():
    # there's no need to expose the entire client...
    app = Flask(__name__)
    # initialize the config of the app object
    app.config.from_pyfile('config/settings.cfg')

    # move into the app context and initialize the amberdata project
    with app.app_context():
        addepar.init_app(app)
        yield addepar.request

def test_version(addepar_request, requests_mock):
    requests_mock.get("https://maffay.addepar.com/api/v1/api_version", json={'a': 'b'})
    y = addepar_request.version
    assert y == {'a': 'b'}

def test_auth(addepar_request):
    assert addepar_request.auth == ("A", "B")

def test_dicturl(addepar_request):
    x = addepar_request.dicturl({"a": 200, "b": "cc"})
    assert x == "a=200&b=cc"

def test_view_csv(addepar_request, requests_mock):
    xxx = pd.DataFrame(index=["x", "y"], columns=["a"], data=[[4], [5]])
    xxx.index.name = "Peter"

    requests_mock.get(
        "https://maffay.addepar.com/api/v1/portfolio/views/10/results?portfolio_id=20&portfolio_type=entity&output_type=csv&start_date=1978-11-12&end_date=1978-11-12",
        text=xxx.to_csv())
    x = addepar_request.view_csv(view_id=10, portfolio_id=20, portfolio_type=PortfolioType.ENTITY,
                           start_date=pd.Timestamp("12-Nov-1978"), end_date=pd.Timestamp("12-Nov-1978"))

    pdt.assert_frame_equal(x, xxx.reset_index())

def test_users(addepar_request,requests_mock):
    requests_mock.get("https://maffay.addepar.com/api/v1/users",
                      json={"data": [{"id": "1", "attributes": {"a": 2.0, "b": 3.0}}], "links": {"next": None}})
    x = {key: attr for key, attr in addepar_request.users}
    assert x == {"1": {"a": 2.0, "b": 3.0}}

def test_user_id(addepar_request,requests_mock):
    requests_mock.get("https://maffay.addepar.com/api/v1/users/1",
                      json={"data": {"id": "1", "attributes": {"a": 2.0, "b": 3.0}}})
    x = addepar_request.user(id=1)
    assert x == {"a": 2.0, "b": 3.0}

def test_entities(addepar_request,requests_mock):
    requests_mock.get("https://maffay.addepar.com/api/v1/entities",
                      json={"data": [{"id": "1", "attributes": {"a": 2.0, "b": 3.0}}], "links": {"next": None}})
    x = {key: attr for key, attr in addepar_request.entities()}
    assert x == {"1": {"a": 2.0, "b": 3.0}}

def test_entity_id(addepar_request,requests_mock):
    requests_mock.get("https://maffay.addepar.com/api/v1/entities/1",
                      json={"data": {"id": "1", "attributes": {"a": 2.0, "b": 3.0}}})
    x = addepar_request.entity(id=1)
    assert x == {"a": 2.0, "b": 3.0}

def test_group_id(addepar_request,requests_mock):
    requests_mock.get("https://maffay.addepar.com/api/v1/groups/1",
                      json={"data": {"id": "1", "attributes": {"a": 2.0, "b": 3.0}}})
    x = addepar_request.group(id=1)
    assert x == {"a": 2.0, "b": 3.0}

def test_options(addepar_request):
    with patch.object(AddeparRequest, "entities") as mock:
        addepar_request.options
        mock.assert_called_once_with(link="/v1/entities", modeltype="OPTION")

def test_persons(addepar_request):
    with patch.object(AddeparRequest, "entities") as mock:
        addepar_request.persons
        mock.assert_called_once_with(link="/v1/entities", modeltype="PERSON_NODE")

def test_groups(addepar_request):
    with patch.object(AddeparRequest, "entities") as mock:
        addepar_request.groups
        mock.assert_called_once_with(link="/v1/groups")

def test_users(addepar_request):
    with patch.object(AddeparRequest, "entities") as mock:
        addepar_request.users
        mock.assert_called_once_with(link="/v1/users")

def test_delete(addepar_request):
    with patch("requests.delete") as mock:
        mock.return_value = True
        assert addepar_request.delete(entity="1")

def test_post(addepar_request):
    with patch("requests.post") as mock:
        mock.return_value = AttrDict({"ok": True, "status_code": 200})
        assert addepar_request.post(data=1)


import numpy
import pickle
import math
from tqdm import tqdm

def average(x):
    return float(sum(x)) / len(x)

def cosine_similarity(v1, v2):
    """
    Calculate the cosine similarity between two vectors v1 and v2.
    The formula for cosine similarity is (v1 dot v2) / (||v1||*||v2||).
    """
    dot_product = sum([x*y for x,y in zip(v1, v2)])
    magnitude_v1 = math.sqrt(sum([x*x for x in v1]))
    magnitude_v2 = math.sqrt(sum([y*y for y in v2]))

    if (magnitude_v1 == 0 or magnitude_v2 == 0):
        return 0

    return dot_product / (magnitude_v1 * magnitude_v2)

def pearson(x, y):
    """
    Calculate the Pearson correlation coefficient between two lists x and y.
    The formula for Pearson correlation coefficient is (sum((x-avg(x))*(y-avg(y)))) / sqrt(sum((x-avg(x))^2) * sum((y-avg(y))^2)).
    """
    n = len(x)
    if n == 0:
        return -1
    avg_x = sum(x) / n
    avg_y = sum(y) / n
    diffprod = sum([(xi - avg_x) * (yi - avg_y) for xi, yi in zip(x, y)])
    xdiff2 = sum([(xi - avg_x)**2 for xi in x])
    ydiff2 = sum([(yi - avg_y)**2 for yi in y])
    if xdiff2 == 0 or ydiff2 == 0:
        return -1
    return diffprod / math.sqrt(xdiff2 * ydiff2)

curr_file_user = 1

user_1_base_file = open(f"ml-100k/u{curr_file_user}.base","r")
user_1_test_file = open(f"ml-100k/u{curr_file_user}.test","r")

user_1_base = user_1_base_file.readlines()
user_1_test = user_1_test_file.readlines()

user_1_base_file.close()
user_1_test_file.close()

user = 943
item = 1682

full_table = [[0 for x in range(item+1)] for y in range(user+1)]
avg_rating = [0 for x in range(user+1)] ## for storing the average of ratings

for line in user_1_base:
    splitted_arr = line.split()
    full_table[int(splitted_arr[0])][int(splitted_arr[1])] = int(splitted_arr[2])

for j in range(1,user+1):
    total_sum = sum(full_table[j])
    total_ratings = item - (full_table[j].count(0)-1)
    avg_rating[j] = float(total_sum)/total_ratings

user_avg = []
user_var = []
user_omega = []
user_baseline = []

for i in range(1,user+1):
    user_avg.append(average(full_table[i]))

for i in tqdm(range(1,user+1)):
    var = 0
    n = 0
    for j in range(1,item+1):
        if (full_table[i][j]!=0):
            var+= (user_avg[i-1] - full_table[i][j])**2
            n+=1
    if (n>1):
        var = var/(n-1)

    user_var.append(var)

max_var = max(user_var)
min_var = min(user_var)

for i in range(1,user+1):
    user_omega.append((user_var[i-1]-min_var)/max_var)

for i in range(1,user+1):
    user_mean = average(full_table[i])
    curr = []
    for j in full_table[i]:
        curr.append(j-user_mean)
    user_baseline.append(curr)

user_top_neighbours=[]

##### Creating Similarity Matrix for users ####
user_sim = [[0 for x in range(user+1)] for y in range(user+1)]
for u1 in range(1,user+1):
    curr = []
    for u2 in range(u1+1,user+1):
        user1_arr = []
        user2_arr = []
        for i in range(1,item+1):
            if full_table[u1][i] != 0 and full_table[u2][i] != 0:
                user1_arr.append(full_table[u1][i])
                user2_arr.append(full_table[u2][i])

        curr_sim = cosine_similarity(user1_arr,user2_arr)

        user_sim[u1][u2] = curr_sim
        user_sim[u2][u1] = curr_sim

        curr.append([u2,curr_sim])

    curr = sorted(curr,key=lambda l:l[1], reverse=True)
    user_top_neighbours.append(curr)

K = 10

totalerror = 0
testsetlen = len(user_1_test)

for line in user_1_test:
    splitted_arr = line.split()
    user_1_id = int(splitted_arr[0])
    item_id = int(splitted_arr[1])
    curr_rating = int(splitted_arr[2])

    score = 0
    tot_sim = 0

    top_n = user_top_neighbours[user_1_id-1]

    active_z = []
    for j in range(item):
        active_z.append(user_baseline[user_1_id-1][j]*user_omega[user_1_id-1])

    j = 0
    t = 0
    while j<K and t<len(top_n):
        curr = top_n[t]

        user_2 = curr[0]
        similarity = curr[1]

        if full_table[user_2][item_id]!=0:
            new_weight = pearson(active_z,user_baseline[user_2-1])
            total_weight = new_weight*similarity

            score += (full_table[user_2][item_id]-avg_rating[user_2]) * total_weight
            tot_sim += similarity * user_omega[user_2-1]

            j+=1

        t+=1

    if tot_sim == 0:
        testsetlen -= 1
        continue

    pred_rating = int(round(float(score)/tot_sim +avg_rating[user_1_id]))
    totalerror += abs(curr_rating - pred_rating)

mae = float(totalerror)/testsetlen
print (f"MAE = {mae}")


import os
import functools
from time import perf_counter

import torch
from torch.utils.tensorboard import SummaryWriter
import transformers

class AverageMeter(object):
    """Computes and stores the average and current value"""

    def __init__(self, name, fmt=':f'):
        self.name = name
        self.fmt = fmt
        self.reset()

    def reset(self):
        self.val = 0
        self.avg = 0
        self.sum = 0
        self.count = 0

    def update(self, val, n=1):
        self.val = val
        self.sum += val * n
        self.count += n
        self.avg = self.sum / self.count

    def __str__(self):
        fmtstr = '{name} {val' + self.fmt + '} ({avg' + self.fmt + '})'
        return fmtstr.format(**self.__dict__)

def compute_metrics(metrics: dict, preds: torch.Tensor, labels: torch.Tensor):
    results = {}
    for metric_name, metric in metrics.items():
        results[metric_name] = metric(preds, labels).item()

    return results

def reset_trackers(trackers: dict):
    for metric in trackers.values():
        metric.reset()

def update_trackers(trackers: dict, metrics_results: dict):
    for metric_name, metric in trackers.items():
        metric.update(metrics_results[metric_name])

def get_trackers_avg(trackers: dict):
    avg_results = {}
    for metric_name, metric in trackers.items():
        avg_results[metric_name] = metric.avg

    return avg_results

def write_to_tensorboard(writer: SummaryWriter, logs: dict, mode: str, step: int):
    for metric_name, value in logs.items():
        writer.add_scalar(f'{mode}/{metric_name}', value, step)

def maybe_save_checkpoint(model, path, epoch_num, save_freq):
    """
    Save a checkpoint
    Args:
        model: a nn.Module instance
        path: path to save checkpoint to
        epoch_num: current epoch number
        save_freq: save frequency based on epoch number

    """
    if not os.path.exists(path):
        os.makedirs(path)
    path = os.path.join(path, f'{epoch_num}.pt')
    if epoch_num % save_freq == 0:
        checkpoint = {'model': model.state_dict(),
                      'epoch': epoch_num}
        torch.save(checkpoint, path)
        print(f'Saved checkpoint to `{path}`')

def get_lm_module_name(model):
    """
    Return the name of the LM part e.g, distilbert, bert, roberta since we don't know which language model is used
    when loading model using AutoModel
    """
    modules = list(model.named_modules())[1:]  # ignore the main module
    for name, m in modules:
        if issubclass(m.__class__, transformers.PreTrainedModel):
            return name

def time_it(method):
    @functools.wraps(method)
    def inner(self, *args, **kwargs):
        start = perf_counter()
        out = method(self, *args, **kwargs)
        if type(out) != tuple:
            out = out,
        end = perf_counter()
        return tuple([*out, end - start])

    return inner

def clean_text(text, invalid_chars):
    text_ = text.split()
    text_ = [word for word in text_ if not word.isascii()]
    cleaned = ''
    valid = True
    for i, word in enumerate(text_):
        for x in invalid_chars:
            if x in word:
                valid = False
                break
        if valid:
            cleaned += f'{word} '
    return cleaned.strip()


import cv2 as cv
import os
from imutils import paths
import shutil

def listPosImagem():

    # Entra na pasta contendo as imagens positivas
    imagemPath = list(paths.list_images('Imagens/comMascara'))
    numero = 1

    # Cria uma pasta chamada 'pos'
    if not os.path.exists('pos'):
        os.makedirs('pos')

    # Percorre as imagens da pasta e cria uma imagem formatada em escala cinza
    for foto in imagemPath:

        foto.replace(foto, f"pos/{numero}.png")
        shutil.copy(foto, foto.replace(foto, f"pos/{numero}.png"))
        img = cv.imread(f"pos/{numero}.png", cv.IMREAD_GRAYSCALE)
        resized_image = cv.resize(img, (100, 100))
        cv.imwrite(f"pos/{numero}.png", resized_image)

        print(foto.replace(foto, f"pos/{str(numero)}.png"))

        numero += 1

# Chama a função
listPosImagem()


import requests
from bs4 import BeautifulSoup
from PIL import Image
from io import BytesIO
donatelink = 'https://act.biologicaldiversity.org/onlineactions/mLBxIVrBY0-PF8_oJWJbEw2?utm_expid=.QGpwm5uyRCWKJOr0upwtRQ.2&utm_referrer=https%3A%2F%2Fwww.biologicaldiversity.org%2F'
home = requests.get('https://www.biologicaldiversity.org/species/')
soup = BeautifulSoup(home.text, 'lxml')
mainlinks = soup.find(class_ = 'col cbd_content_main').find_all('a')
speclist = [spec.text for spec in mainlinks]
refs = ['https://www.biologicaldiversity.org/species/' + link['href'] for link in mainlinks]

def userprogram():

    for a, b in enumerate(speclist, 1):
        print("{}. {}".format(a, b))
    i = int(input('Type a number 1-7 to explore a subject: '))
    page = requests.get(refs[i-1])
    soup2 = BeautifulSoup(page.text, 'lxml')
    species_img_url = soup2.find(class_='banner').find('img')['src']
    species_img_link = 'https://www.biologicaldiversity.org/' + species_img_url
    species_pic = requests.get(species_img_link)
    species_img = Image.open(BytesIO(species_pic.content))
    species_img.show()
    speciesinfo = [paragraph.text for paragraph in soup2.find(class_= 'col cbd_content_main').find_all('p')]
    print(speclist[i-1])
    for info in speciesinfo:
        if len(info) > 5:
            print(info)
            print("-"*20)
        else:
            pass
    ii = input('Would you like a list of {} that you could help to save? (type y/n) '.format(speclist[i-1]))
    if ii.lower() == 'y':
        species_img.close()
        creature_page = soup2.find(class_='col-md-4 cbd_content_sidebar').find_all('a')
        creature_links = ['https://www.biologicaldiversity.org/species/' + speclist[i-1].lower() + '/' + creature['href'] for creature in creature_page if len(creature.text) > 2]
        creature_list = [creature.text for creature in creature_page if len(creature.text) > 2]
        for a, b in enumerate(creature_list, 1):
            print('{}. {}'.format(a, b.title()))
        print('Type a number, 1-{} and press enter to view'.format(len(creature_list)))
        iii = int(input(''))
        creature_site = requests.get(creature_links[iii-1])
        creature_soup = BeautifulSoup(creature_site.text, 'lxml')
        creature_summary = creature_soup.find(class_='col-md-6 cbd_content_left').find_all('p')
        creature_img_url = creature_soup.find(class_='banner').find('img')['src']
        creature_img_link = 'https://www.biologicaldiversity.org/' + creature_img_url
        r = requests.get(creature_img_link)
        species_img = Image.open(BytesIO(r.content))
        species_img.show(title='creature_list[iii-1]')
        print('-'*20)
        print('Here\'s a pic of the {}'.format(creature_list[iii-1]))
        print('-'*20)

        for sec in creature_summary:
            print(sec.text)
            print('-'*20)
        print('The {} needs your help!  Click this link to donate: '.format(creature_list[iii-1]))
        print(donatelink)

userprogram()


import numpy as np
import pandas as pd
import seaborn as sns
from matplotlib.axes import Axes
from matplotlib import pyplot as plt

def plot_kde(data, xlabel, ylabel, title, clip):
	sns.set_style("darkgrid")
	plot: Axes = sns.kdeplot(data=data,
	                         shade=True,
	                         kernel="gau",
	                         clip=clip,
	                         legend=False,
	                         color=sns.xkcd_rgb["dark red"])
	plot.set_title(title, fontdict={"size": 14})
	plot.set_xlabel(xlabel)
	plot.set_ylabel(ylabel)

df: pd.DataFrame = pd.read_csv("final_dns_dataset.csv", usecols=[5,9,10,11,12,13])
df.rename({"authoritative_state": "state", "authoritative_latitude": "lat", "authoritative_longitude": "lng"}, inplace=True, axis=1)
df = df.groupby(["lat", "lng"], as_index=False).agg({"state": "first", "rtt": ["median", "mean", "std"], "rtt_normalized": ["median", "mean", "std"]})

df = df["rtt_normalized"]["median"]
quantiles = df.quantile([0.025, 0.975]).to_numpy()
plot_kde(data=df,
         xlabel="Normalized RTT median per location (ms)",
         ylabel="Density",
         title="Distribution of DNS Normalized RTT Median by Location",
         clip=quantiles)
plt.show()


# 視圖 - 直接面對使用者的使用者介面設計
from flask import Flask, request, jsonify, render_template

from app import app, db
from models import Record

@app.route("/")
def index():
    return render_template('index.html', title='To-Do List')

@app.route("/record", methods=['POST'])
def add_record():
    req_data = request.form
    todo = req_data['name']
    date = req_data['cost']
    record = Record(todo=todo, date=date)
    db.session.add(record)
    db.session.commit()
    return 'Create Succeeded', 200

@app.route("/record", methods=['GET'])
def get_records():
    records = Record.query.all()
    records_data = [
        {
            'id': record.id,
            'name': record.todo,
            'cost': record.date
        }
        for record in records
    ]
    return jsonify(records_data), 200

@app.route('/record/<int:record_id>', methods=['GET'])
def get_record(record_id):
    record = Record.query.filter_by(id=record_id).first()
    record_data = {
        'id': record.id,
        'name': record.todo,
        'cost': record.date
    }
    return jsonify(record_data), 200

@app.route('/record/<int:record_id>', methods=['PUT'])
def update_record(record_id):
    req_data = request.form
    record = Record.query.filter_by(id=record_id).first()
    record.todo = req_data['name']
    record.date = req_data['cost']
    db.session.add(record)
    db.session.commit()
    return 'Update Succeeded', 200

@app.route("/record/<int:record_id>", methods=["DELETE"])
def delete_record(record_id):
    record = Record.query.filter_by(id=record_id).first()
    db.session.delete(record)
    db.session.commit()
    return 'Delete Succeeded', 200


"""LSST LSST the Docs and DocuShare/ls.st reference roles."""

from __future__ import annotations

from typing import TYPE_CHECKING, Dict, List, Optional, Tuple

from docutils import nodes

if TYPE_CHECKING:
    from docutils.nodes import Node, system_message
    from docutils.parsers.rst.states import Inliner
    from sphinx.application import Sphinx

def lsstio_doc_shortlink_role(
    name: str,
    rawtext: str,
    text: str,
    lineno: int,
    inliner: Inliner,
    options: Optional[Dict] = None,
    content: Optional[List[str]] = None,
) -> Tuple[List[Node], List[system_message]]:
    """Link to LSST documents given their handle that are hosted on
    lsst.io (Rubin's deployment of LSST the Docs).

    Example::

        :sqr:`001`
    """
    options = options or {}
    content = content or []
    node = nodes.reference(
        text="{0}-{1}".format(name.upper(), text),
        refuri="https://{0}-{1}.lsst.io/".format(name.lower(), text),
        **options,
    )
    return [node], []

def lsst_doc_shortlink_role(
    name: str,
    rawtext: str,
    text: str,
    lineno: int,
    inliner: Inliner,
    options: Optional[Dict] = None,
    content: Optional[List[str]] = None,
) -> Tuple[List[Node], List[system_message]]:
    """Link to LSST documents given their handle using LSST's ls.st link
    shortener.

    Example::

        :ldm:`151`
    """
    options = options or {}
    content = content or []
    node = nodes.reference(
        text="{0}-{1}".format(name.upper(), text),
        refuri="https://ls.st/{0}-{1}".format(name, text),
        **options,
    )
    return [node], []

def lsst_doc_shortlink_titlecase_display_role(
    name: str,
    rawtext: str,
    text: str,
    lineno: int,
    inliner: Inliner,
    options: Optional[Dict] = None,
    content: Optional[List[str]] = None,
) -> Tuple[List[Node], List[system_message]]:
    """Link to LSST documents given their handle using LSST's ls.st link
    shortener with the document handle displayed in title case.

    This role is useful for Document, Report, Minutes, and Collection
    DocuShare handles.

    Example::

        :document:`1`
    """
    options = options or {}
    content = content or []
    node = nodes.reference(
        text="{0}-{1}".format(name.title(), text),
        refuri="https://ls.st/{0}-{1}".format(name, text),
        **options,
    )
    return [node], []

def setup(app: Sphinx) -> None:
    # LSST Data Management
    app.add_role("ldm", lsst_doc_shortlink_role)
    # LSST Systems Engineering
    app.add_role("lse", lsst_doc_shortlink_role)
    # LSST Project Management
    app.add_role("lpm", lsst_doc_shortlink_role)
    # LSST Telescope & Site
    app.add_role("lts", lsst_doc_shortlink_role)
    # LSST Education & Public Outreach
    app.add_role("lep", lsst_doc_shortlink_role)
    # LSST Camera System
    app.add_role("lca", lsst_doc_shortlink_role)
    # LSST Board
    app.add_role("lsstc", lsst_doc_shortlink_role)
    # LSST Change Request
    app.add_role("lcr", lsst_doc_shortlink_role)
    # LSST Camera Change Notice
    app.add_role("lcn", lsst_doc_shortlink_role)
    # LSST Operations
    app.add_role("lso", lsst_doc_shortlink_role)
    # LSST Data Management Test Report
    app.add_role("dmtr", lsst_doc_shortlink_role)
    # LSST Document
    app.add_role("document", lsst_doc_shortlink_titlecase_display_role)
    # LSST Report
    app.add_role("report", lsst_doc_shortlink_titlecase_display_role)
    # LSST Minutes
    app.add_role("minutes", lsst_doc_shortlink_titlecase_display_role)
    # DocuShare collection
    app.add_role("collection", lsst_doc_shortlink_titlecase_display_role)

    # Technical notes are hosted canonically on www.lsst.io

    # LSST SQuaRE Technical Note
    app.add_role("sqr", lsstio_doc_shortlink_role)
    # LSST Data Management Technical Note
    app.add_role("dmtn", lsstio_doc_shortlink_role)
    # LSST Simulations Technical Note
    app.add_role("smtn", lsstio_doc_shortlink_role)
    # SITCOMTN Technical Note
    app.add_role("sitcomtn", lsstio_doc_shortlink_role)
    # PST Technical Note
    app.add_role("pstn", lsstio_doc_shortlink_role)
    # Rubin Technical Note
    app.add_role("rtn", lsstio_doc_shortlink_role)
    # IT Technical Note
    app.add_role("ittn", lsstio_doc_shortlink_role)
    # T&S Technical Note
    app.add_role("tstn", lsstio_doc_shortlink_role)


# pylint: disable=import-error
from collections import Counter
import numpy as np
import pandas as pd
import streamlit as st
from transformers import AutoTokenizer
import tokenizers
import random
import sys
sys.path.append('/home/matane/matan/dev/WSIatScale')

from WSIatScale.analyze import (read_files,
                                RepInstances,
                                prepare_arguments,
                                tokenize)
from WSIatScale.clustering import ClusterFactory
from WSIatScale.community_detection import CommunityFinder
from WSIatScale.apriori import run_apriori

from utils.utils import StreamlitTqdm
import altair as alt

import networkx as nx
import matplotlib.pyplot as plt
from PIL import Image

SEED = 111

@st.cache(suppress_st_warning=True, allow_output_mutation=True)
def cached_read_files_specific_n_reps(token, data_dir, sample_n_files, full_stop_index, n_reps):
    rep_instances, msg  = read_files(token, data_dir, sample_n_files, full_stop_index)#, bar=StreamlitTqdm)
    rep_instances.populate_specific_size(n_reps)
    return rep_instances, msg

@st.cache(hash_funcs={tokenizers.Tokenizer: id}, suppress_st_warning=True, allow_output_mutation=True)
def cached_tokenizer(model_hf_path):
    tokenizer = AutoTokenizer.from_pretrained(model_hf_path, use_fast=True)
    return tokenizer

@st.cache(hash_funcs={RepInstances: id}, suppress_st_warning=True, allow_output_mutation=True)
def cached_CommunityFinder(rep_instances):
    return CommunityFinder(rep_instances)

@st.cache(hash_funcs={CommunityFinder: id}, suppress_st_warning=True, allow_output_mutation=True)
def cached_find_communities(community_finder, method):
    communities = community_finder.find(method)
    return communities

@st.cache(hash_funcs={CommunityFinder: id, tokenizers.Tokenizer: id}, suppress_st_warning=True, allow_output_mutation=True)
def cached_create_graph(community_finder, communities, tokenizer):
    create_graph(community_finder, communities, tokenizer)

def main():
    st.title('WSI at Scale')

    dataset = st.sidebar.selectbox('Dataset', ('Wikipedia', 'CORD-19', 'SemEval2010', 'SemEval2013'), 0)
    args = prepare_arguments()
    example_word, full_stop_index = dataset_configs(dataset, args)

    tokenizer = cached_tokenizer(args.model_hf_path)
    half_words_list = np.load(f"non-full-words/non-full-words-{args.model_hf_path}.npy")

    word = st.sidebar.text_input('Word to disambiguate: (Split multiple words by `;` no space)', example_word)
    n_reps = st.sidebar.slider(f"Number of replacements (Taken from {args.model_hf_path}'s masked LM)", 1, 100, 20)
    args.n_reps = n_reps

    sample_n_files = st.sidebar.number_input("Number of files", min_value=1, max_value=10000, value=1000)
    args.sample_n_files = sample_n_files
    if word == '': return

    rep_instances = None
    for w in word.split(';'):
        token = None
        args.word = w

        try:
            token = tokenize(tokenizer, w) if dataset != 'SemEval2010' else w
        except ValueError as e:
            st.write(f'Word {w} is more than a single wordpiece. Please choose a different word.')

        if token:
            curr_word_rep_instances = read_files_from_cache(args, token, n_reps, sample_n_files, full_stop_index)
            curr_word_rep_instances.remove_certain_words(word=w,
                                                         tokenizer=tokenizer,
                                                         remove_query_word=True,
                                                         half_words_list=half_words_list)

            if rep_instances is None:
                rep_instances = curr_word_rep_instances
            else:
                rep_instances.merge(curr_word_rep_instances)

    action = st.sidebar.selectbox(
        'How to Group Instances',
        ('Select Action', 'Group by Communities', 'Cluster', 'Apriori'),
        index=1)

    if rep_instances and action == 'Cluster':
        cluster_alg, n_sents_to_print = prepare_choices(args)
        display_clusters(args, tokenizer, rep_instances, cluster_alg, n_sents_to_print)

    if rep_instances and action == 'Group by Communities':
        display_communities(tokenizer, rep_instances)

    if rep_instances and action == 'Apriori':
        display_apriori(tokenizer, rep_instances)

def dataset_configs(dataset, args):
    if dataset == 'Wikipedia':
        model = st.sidebar.selectbox('Model', ('RoBERTa', 'BERT'), 1)
        if model == 'RoBERTa':
            args.model_hf_path = 'roberta-large'
            args.data_dir = '/mnt/disks/mnt1/datasets/processed_for_WSI/wiki/all'
            example_word = ' bass'
            full_stop_index = 4
        else:
            args.model_hf_path = 'bert-large-cased-whole-word-masking'
            args.data_dir = '/mnt/disks/mnt2/datasets/processed_for_WSI/wiki/bert/'
            example_word = 'bass'
            full_stop_index = 119
    elif dataset == 'CORD-19':
        args.model_hf_path = 'allenai/scibert_scivocab_uncased'
        args.data_dir = '/mnt/disks/mnt1/datasets/processed_for_WSI/CORD-19'
        example_word = 'race'
        full_stop_index = 205
    elif dataset == 'SemEval2010':
        args.model_hf_path = 'bert-large-uncased'
        # args.data_dir = '/home/matane/matan/dev/WSIatScale/write_mask_preds/out/SemEval2010/bert-large-uncased'
        args.data_dir = '/home/matane/matan/dev/WSIatScale/write_mask_preds/out/SemEval2010/bert-large-uncased-no-double-instances'
        example_word = 'officer'
        full_stop_index = None
    elif dataset == 'SemEval2013':
        args.model_hf_path = 'bert-large-uncased'
        args.data_dir = '/home/matane/matan/dev/WSIatScale/write_mask_preds/out/SemEval2013/bert-large-uncased'
        example_word = 'become'
        full_stop_index = None
    return example_word, full_stop_index

def read_files_from_cache(args, token, n_reps, sample_n_files, full_stop_index):
    try:
        rep_instances, msg = cached_read_files_specific_n_reps(token, args.data_dir, sample_n_files, full_stop_index, n_reps)
        st.write(msg)
        return rep_instances
    except ValueError as e:
        st.write(e)

def display_clusters(args, tokenizer, rep_instances, cluster_alg, n_sents_to_print):
    clustering_load_state = st.text('Clustering...')

    model = ClusterFactory.make(cluster_alg, args)
    clusters = model.fit_predict(rep_instances)
    clustered_reps = model.reps_to_their_clusters(clusters, rep_instances)
    representative_sents = model.representative_sents(clustered_reps, n_sents_to_print)

    st.header(f"Found {len(set(clusters))} clusters.")
    for words_in_cluster, sents_data, msg in model.group_for_display(args, tokenizer, clustered_reps, representative_sents):
        st.subheader(msg['header'])
        st.write(msg['found'])
        if words_in_cluster:
            keys, values = zip(*words_in_cluster)
            source = pd.DataFrame({'words': keys, 'occurrences': values})

            chart = alt.Chart(source).mark_bar().encode(
                x='occurrences:Q',
                y=alt.Y('words:N', sort='-x')
            ).configure_axis(
                labelFontSize=13,
            )
            chart.configure_axis(labelFontSize=0)
            st.altair_chart(chart, use_container_width=True)
            if n_sents_to_print > 0:
                st.write('**Exemplary Sentences**')
                for sent_data in sents_data:
                    st.write(f"* {tokenizer.decode(sent_data.sent).lstrip()}")

    clustering_load_state.text('')

def display_communities(tokenizer, rep_instances):
    community_alg = st.selectbox(
        'Community Algorithm',
        ('louvain', ),
        index=0)

    n_sents_to_print = st.number_input('Exemplary Sentences to Present', value=3, min_value=0)
    at_least_n_matches = st.number_input('Number of Minimum Sentence Matches', value=1, min_value=0, max_value=100)
    community_finder = cached_CommunityFinder(rep_instances)
    communities = cached_find_communities(community_finder, community_alg)

    communities_tokens, communities_sents_data = community_finder.argmax_voting(communities, rep_instances)

    print_communities(tokenizer,
                      community_finder,
                      communities,
                      communities_tokens,
                      communities_sents_data,
                      n_sents_to_print,
                      at_least_n_matches)

def print_communities(tokenizer,
                      community_finder,
                      communities,
                      communities_tokens,
                      communities_sents_data,
                      n_sents_to_print,
                      at_least_n_matches):
    num_skipped = 0
    random.seed(SEED)
    for comm, rep_instances in zip(communities_tokens, communities_sents_data):
        if len(rep_instances) < at_least_n_matches:
            num_skipped += 1
            continue
        random.shuffle(rep_instances)
        checkbox_text = get_checkbox_text(comm, rep_instances, tokenizer)
        display_sents = st.checkbox(checkbox_text + f" - ({len(rep_instances)} sents)")
        if display_sents:
            for rep_inst in rep_instances[:n_sents_to_print]:
                text = f"{tokenizer.decode(rep_inst.sent).lstrip()}"
                reps_text = " ".join([tokenizer.decode([rep]).lstrip() for rep in rep_inst.reps])
                st.write(f"* **{reps_text}:** ", text)
    if num_skipped > 0:
        st.write(f"Skipped {num_skipped} communities with less than {at_least_n_matches} sentences.")

    cached_create_graph(community_finder, communities, tokenizer)
    print_graph()

def get_checkbox_text(comm, rep_instances, tokenizer):
    max_words_to_display = 10
    community_tokens_counter = {t: 0 for t in comm}
    for rep_inst in rep_instances:
        for token in comm:
            if token in rep_inst.reps:
                community_tokens_counter[token] += 1
    community_tokens_counter = {k: v for k, v in sorted(community_tokens_counter.items(), key=lambda item: item[1], reverse=True)}

    checkbox_text = " ".join([tokenizer.decode([t]) for t in list(community_tokens_counter.keys())[:max_words_to_display]])
    if len(community_tokens_counter) > max_words_to_display:
        checkbox_text = checkbox_text+" ..."
    return checkbox_text

def create_graph(community_finder, communities, tokenizer):

    original_G = nx.from_numpy_matrix(community_finder.cooccurrence_matrix)
    components = list(nx.connected_components(original_G))
    nodes_of_largest_component  = max(components, key = len)
    G = original_G.subgraph(nodes_of_largest_component)
    degrees = dict(G.degree)
    largest_degree = max(degrees.values())

    pos = nx.spring_layout(G, k=0.1, iterations=50, seed=SEED)
    # pos = nx.spring_layout(G, k=0.9, iterations=20, seed=SEED)
    plt.axis('off')
    partition = {n:c for c in range(len(communities)) for n in communities[c]}
    sorted_partition_values = [partition[n] for n in G.nodes()]
    nx.draw_networkx_nodes(G,
                           pos,
                           node_size=[float(v)/largest_degree*500 for v in degrees.values()],
                           cmap=plt.cm.Set3,
                           node_color=sorted_partition_values)
    nx.draw_networkx_edges(G,
                           pos,
                           alpha=0.3)
    if G.number_of_edges() < 50:
        weights = nx.get_edge_attributes(G, 'weight')
        weights = {k: int(v) for k, v in weights.items()}
        nx.draw_networkx_edge_labels(G, pos, font_size=6, edge_labels=weights)

    if len(G.nodes()) <= 50:
        node2word = {n: tokenizer.decode([community_finder.node2token[n]]) for n in G.nodes()}
    else:
        most_common_nodes, _ = zip(*Counter(degrees).most_common(20))
        node2word = {n: tokenizer.decode([community_finder.node2token[n]]) if n in most_common_nodes else '' for n in G.nodes()}
    nx.draw_networkx_labels(G, pos, node2word, font_size=8)

    plt.savefig("/tmp/Graph.png", format="PNG")
    if len(components) > 1:
        st.write(f"Additional {len(components) - 1} components not presented.")

def print_graph():
    image = Image.open('/tmp/Graph.png')
    st.image(image, use_column_width=True)

def prepare_choices(args):
    cluster_alg = st.selectbox(
        'Clustering Algorithm',
        ('BOW Hierarchical', ),
        index=0)

    args.show_top_n_words_per_cluster = st.slider('Number of words to show per cluster', 1, 100, 10)
    n_sents_to_print = st.number_input('Exemplary Sentences to Present', value=1, min_value=0)
    args.n_sents_to_print = n_sents_to_print
    return cluster_alg, n_sents_to_print

def display_apriori(tokenizer, rep_instances):
    min_support = st.slider('Min Support: Frequency of which the items in the rule appear together in the data set.', 0.0, 1.0, 0.5)
    itemsets = run_apriori(rep_instances, min_support)
    st.subheader('Apriori Itemsets')
    if max(itemsets) != 1:
        itemset_sizes = st.slider("itemsets sizes", 1, max(itemsets), (1, max(itemsets)))
    else:
        itemset_sizes = (1, 1)

    for size in range(itemset_sizes[1], itemset_sizes[0]-1, -1):
        itemset_of_size = itemsets[size]
        sorted_itemset_of_size = {k: v.itemset_count for k, v in sorted(itemset_of_size.items(), key=lambda item: item[1].itemset_count, reverse=True)}
        for i, (item, frequency) in enumerate(sorted_itemset_of_size.items()):
            st.write(f"{tokenizer.decode(list(item))}: {frequency}")
            if i > 9:
                st.write(f"**{len(itemsets[size])-9} additional itemsets are available of size {size}**")
                break

if __name__ == "__main__":
    main()


from django.shortcuts import render

def django(request):
    """A view that renders a django template."""
    return render(
        request,
        'superadvanced/django.html',
        {'title': 'This is a Django template #superadvanced'})

def jinja(request):
    """A view that renders a jinja template."""
    return render(
        request,
        'superadvanced/jinja.html',
        {'title': 'This is a Jinja2 template #superadvanced'})