'''
Created on Jan 19, 2017

@author: Alexandre Day

    Purpose:
        Compute robustness of clustering for indivial data point

'''

import numpy as np
import pickle
from matplotlib import pyplot as plt
import seaborn as sns
import pandas as pd
from sklearn import datasets
import time
from density_cluster import DCluster

def main():

    n_true_center=15
    n_sample=10000
    X,y=datasets.make_blobs(n_sample,2,n_true_center,random_state=0)

    dcluster=DCluster()
    idx_sub=np.sort(np.random.choice(range(n_sample), round(0.9*n_sample), replace=False))

    X_sub=X[idx_sub]

    cluster_label_X_sub,idx_centers,rho,delta,kd_tree=dcluster.fit(X_sub)

    """cluster_members is a dictionary from cluster (labelled from [0,n_center[ ) to the absolute idx of it's member points
    """

    cluster_label_X=np.full(n_sample,-1,dtype=np.int32)

    cluster_label_X[idx_sub]=cluster_label_X_sub

    cluster_members=group_by_label(cluster_label_X_sub,idx_sub)

    #print(NH_Cluster(idx_sub[np.random.randint(0,5000,5)],cluster_label_X,X,kd_tree,idx_sub))

def NH_Cluster(a_IDX,cluster_label_X,X,kd_tree,idx_sub):
    """
    Purpose:
        Determines the the neighborhood for data points identified
        by their absolute index in a_IDX array.
    Return:
        List of neighborhood (in absolute index) for each data point in a_IDX.
    """

    n_data=len(a_IDX)
    _,nn_r_IDX=kd_tree.query(list(X[a_IDX]), k=40)
    nn_a_IDX=idx_sub[nn_r_IDX]

    cluster_label_a_IDX=cluster_label_X[a_IDX].reshape(-1,1)

    cluster_label_nn_a_IDX=cluster_label_X[nn_a_IDX]

    tmp=(cluster_label_nn_a_IDX == cluster_label_a_IDX)

    return [nn_a_IDX[i][t] for i,t in zip(range(n_data),tmp)]

    #return cluster_members

def group_by_label(cluster_label,idx_sub):
    """
    Purpose:
        Given the cluster labels found for every data point, return a
        dictionnary from cluster label to absolute indices in that cluster
        The absolute indices are the indices of the data in the whole dataset

    """
    sort_idx = np.argsort(cluster_label)
    a_sorted = cluster_label[sort_idx]
    unq_first = np.concatenate(([True], a_sorted[1:] != a_sorted[:-1]))
    unq_items = a_sorted[unq_first]
    unq_count = np.diff(np.nonzero(unq_first)[0])
    groups=np.split(sort_idx, np.cumsum(unq_count))

    return dict(zip(unq_items,[np.sort(idx_sub[g]) for g in groups]))

#def NH_Cluster(a_IDX,cluster_members,):
#===============================================================================
#
#     idx_sub=np.random.shuffle(range(n_sample))[:round(0.9*n_sample)]
#
#
#
#
#     nn_dist,nn_list=tree.query(list(X), k=40)
#===============================================================================

#===============================================================================
#
#
#
# def NH(a_label,cluster,tree): # a_label refers to absolute label
#===============================================================================

if __name__=="__main__":
    main()


import os
os.environ['OPENBLAS_NUM_THREADS'] = '1'
import numpy as np
import h5py
import json
from scipy.misc import imread, imresize
from tqdm import tqdm
from collections import Counter, defaultdict
from random import seed, sample
import pandas as pd
from stanfordcorenlp import StanfordCoreNLP
from utils import sentence_normalize, un_capitalize, merge_entities, check_exact_author, list2long

nlp = StanfordCoreNLP('/data00/user1/data/corenlp/')
num_topics = 3

def load_annotation(path):
    ret = {}
    anno = json.load(open(path, 'r'))["annotations"]
    for item in anno:
        ret[item["img"]] = {0: item["content"], 1: item["form"], 2: item["context"]}
    return ret

def tokenize(imgs, max_len=200):
    bad_cnt = 0
    for img in tqdm(imgs):
        author = img['author']
        sent_topic_dict = img['sentences']
        token_topic_dict, template_topic_dict = defaultdict(list), defaultdict(list)

        for topic_key in sent_topic_dict.keys():
            sent_tuple = sent_topic_dict[topic_key]
            token_tuple, template_tuple = [], []
            topic = int(topic_key)

            for sent in sent_tuple:
                sent = sentence_normalize(sent)
                doc = nlp.ner(sent)
                tokens = [d[0] for d in doc]
                template = [d[1] + '_' if d[1] != 'O' else d[0] for d in doc]
                try:
                    tokens, template = un_capitalize(tokens, template)
                    assert len(tokens) == len(template), 'length mismatch after normalize %s' % sent
                    assert len(tokens) > 0, "no tokens in the sentence? %s" % sent

                    template = check_exact_author(tokens, template, author, verbose=False)
                    template = merge_entities(template)
                    assert len(template) > 0, "no tokens after merge entities? %s" % sent
                except Exception as e:
                    print(e)
                    bad_cnt += 1
                    print('%d-th bad sentence %s, just skip it' % (bad_cnt, sent))
                    continue
                token_tuple.append(tokens)
                template_tuple.append(template)

            long_token = list2long(token_tuple, max_len)
            long_template = list2long(template_tuple, max_len)
            token_topic_dict[topic] = long_token
            template_topic_dict[topic] = long_template

        img['sentences'] = [{'long_token': token_topic_dict, 'long_template': template_topic_dict}]

    return imgs

def create_dataset(image_folder, output_folder, max_len=100, min_len=5, min_word_freq=20):

    if not os.path.exists(output_folder):
        os.mkdir(output_folder)

    df_train = pd.read_csv('../KnowledgeRetrieval/context_art_classification/Data/SemArt/semart_train.csv',
                           delimiter='\t', encoding='Cp1252')
    df_val = pd.read_csv('../KnowledgeRetrieval/context_art_classification/Data/SemArt/semart_val.csv',
                         delimiter='\t', encoding='Cp1252')
    df_test = pd.read_csv('../KnowledgeRetrieval/context_art_classification/Data/SemArt/semart_test.csv',
                          delimiter='\t', encoding='Cp1252')

    img_names = list(df_train['IMAGE_FILE']) + list(df_val['IMAGE_FILE']) + list(df_test['IMAGE_FILE'])
    titles = list(df_train['TITLE']) + list(df_val['TITLE']) + list(df_test['TITLE'])
    types = list(df_train['TYPE']) + list(df_val['TYPE']) + list(df_test['TYPE'])
    schools = list(df_train['SCHOOL']) + list(df_val['SCHOOL']) + list(df_test['SCHOOL'])
    times = list(df_train['TIMEFRAME']) + list(df_val['TIMEFRAME']) + list(df_test['TIMEFRAME'])
    authors = list(df_train['AUTHOR']) + list(df_val['AUTHOR']) + list(df_test['AUTHOR'])
    sents_train = load_annotation('annotations/semart_topic_annotated_train.json')
    sents_test = load_annotation('annotations/semart_topic_annotated_test.json')

    imgs = []

    for i in tqdm(range(len(img_names))):
        jimg = {}
        jimg['id'] = img_names[i].strip().split('.')[0]
        jimg['filename'] = img_names[i]
        jimg['type'] = types[i]
        jimg['school'] = schools[i]
        jimg['timeframe'] = times[i]
        jimg['author'] = authors[i]
        jimg['title'] = titles[i]
        if img_names[i] in sents_train.keys():
            jimg['sentences'] = sents_train[img_names[i]]
            jimg['split'] = 'train'
        elif img_names[i] in sents_test.keys():
            jimg['sentences'] = sents_test[img_names[i]]
            jimg['split'] = 'test'
        else:
            continue

        imgs.append(jimg)

    imgs = tokenize(imgs, max_len=max_len)

    train_image_paths = []
    train_image_captions = []

    test_image_paths = []
    test_image_captions = []

    word_freq = Counter()

    for img in imgs:
        captions = [[] for _ in range(num_topics)]
        for c in img['sentences']:
            for topic in c['long_template'].keys():
                word_freq.update(c['long_template'][topic])
                if min_len < len(c['long_template'][topic]) <= max_len:
                    captions[int(topic)].append(c['long_template'][topic])

        if len(captions[0]) == 0 and len(captions[1]) == 0 and len(captions[2]) == 0:
            continue

        path = os.path.join(image_folder, img['filename'])

        if img['split'] in {'train'}:
            train_image_paths.append(path)
            train_image_captions.append(captions)
        elif img['split'] in {'test'}:
            test_image_paths.append(path)
            test_image_captions.append(captions)

    # Sanity check
    assert len(train_image_paths) == len(train_image_captions)
    assert len(test_image_paths) == len(test_image_captions)

    # Create word map
    words = [w for w in word_freq.keys() if word_freq[w] > min_word_freq]
    word_map = {k: v + 1 for v, k in enumerate(words)}
    word_map['<unk>'] = len(word_map) + 1
    word_map['<start>'] = len(word_map) + 1
    word_map['<end>'] = len(word_map) + 1
    word_map['<pad>'] = 0
    if '<.>' in word_map.keys():
        print('Word map include stop dot <.>')

    base_filename = 'SemArt_{}_min_word_freq_{}_max_len'.format(min_word_freq, max_len)

    with open(os.path.join(output_folder, 'WORDMAP_' + base_filename + '.json'), 'w') as j:
        json.dump(word_map, j)

    seed(123)
    for impaths, imcaps, split in [(train_image_paths, train_image_captions, 'TRAIN'),
                                   (test_image_paths, test_image_captions, 'TEST')]:
        if len(impaths) == 0:
            continue

        with h5py.File(os.path.join(output_folder, split + '_IMAGES_' + base_filename + '.hdf5'), 'a') as h:
            h.attrs['captions_per_image'] = 1
            images = h.create_dataset('images', (len(impaths), 3, 256, 256), dtype='uint8')
            print("\nReading %s images and captions, storing to file...\n" % split)

            enc_captions = [[] for _ in range(num_topics)]
            caplens = [[] for _ in range(num_topics)]

            for i, path in enumerate(tqdm(impaths)):
                # Sample captions
                assert len(imcaps[i]) == num_topics
                captions = imcaps[i]
                captions_has_content = [x[0] for x in captions if len(x) != 0]
                for t in range(3):
                    if len(captions[t]) == 0:
                        captions[t] = sample(captions_has_content, k=1)

                # Sanity check
                assert len(captions[0]) > 0 and len(captions[1]) > 0 and len(captions[2]) > 0

                # Read images
                img = imread(impaths[i])
                if len(img.shape) == 2:
                    img = img[:, :, np.newaxis]
                    img = np.concatenate([img, img, img], axis=2)
                img = imresize(img, (256, 256))
                img = img.transpose(2, 0, 1)
                assert img.shape == (3, 256, 256)
                assert np.max(img) <= 255
                images[i] = img

                for t in range(num_topics):
                    for j, c in enumerate(captions[t]):
                        # Encode captions
                        enc_c = [word_map['<start>']] + \
                                [word_map.get(word, word_map['<unk>']) for word in c] + \
                                [word_map['<end>']] + \
                                [word_map['<pad>']] * (max_len - len(c))

                        # Find caption lengths
                        c_len = len(c) + 2
                        enc_captions[t].append(enc_c)
                        caplens[t].append(c_len)

            # Sanity check
            assert images.shape[0] * 1 == len(enc_captions[0]) == len(caplens[0])
            assert len(enc_captions[0]) == len(enc_captions[1]) == len(enc_captions[2])
            assert len(caplens[0]) == len(caplens[1]) == len(caplens[2])
            print('Total training image: ', images.shape[0])

            # Save encoded captions and their lengths to JSON files
            with open(os.path.join(output_folder, split + '_CAPTIONS_' + base_filename + '.json'), 'w') as j:
                json.dump(enc_captions, j)

            with open(os.path.join(output_folder, split + '_CAPLENS_' + base_filename + '.json'), 'w') as j:
                json.dump(caplens, j)

if __name__ == '__main__':
    create_dataset(image_folder='../KnowledgeRetrieval/context_art_classification/Data/SemArt/Images',
                   output_folder='dataset_semart',
                   max_len=120,
                   min_len=10,
                   min_word_freq=5)
    nlp.close()


from artiq.tools import simple_network_args
import parser
import argparse
import logging
from artiq.tools import verbosity_args, init_logger
from artiq.tools import bind_address_from_args
from artiq.protocols.pc_rpc import simple_server_loop
import sys

class Controller(object):
    default_port = 3249
    args = None
    name = None

    def __init__(self):
        if self.name is None:
            self.name = self.__class__.__name__
        self.add_global_args()
        self.add_args()
        self.parse_args()
        self.create_logger()
        self.load_drivers()

    def add_global_args(self):
        self.parser = argparse.ArgumentParser(description=self.description)
        self.parser.add_argument(
            "-d", "--device", default=None,
            required=True,
            help="serial port (e.g. COM3).")
        simple_network_args(self.parser, self.default_port)
        verbosity_args(self.parser)

    '''
    Implemented in derived class
    '''
    def add_args(self):
        pass

    def parse_args(self):
        self.args = self.parser.parse_args()
        self.bind_address = bind_address_from_args(self.args)
        self.port = self.args.port

    def create_logger(self):
        self.logger = logging.getLogger(self.name)
        init_logger(self.args)

    def load_drivers(self):
        self.drivers = {}
        for device in self.devices:
            try:
                module = __import__("devices.%s.driver" % device, fromlist=["devices.%s" % device])
                class_ = getattr(module, camel_case(device))
                driver = class_(self.args, self.logger)
                if callable(getattr(driver, "close", None)):
                    driver.load()
                self.drivers[device] = driver
                self.logger.info(" Loaded driver for %s" % (device))
            except ImportError:
                self.logger.error(" Unable to import driver for %s" % device)
            except:
                self.logger.error(" Error occured while importing driver for %s" % device)

    def loop(self):
        try:
            for addr in self.bind_address:
                self.logger.info(" Binding to %s on port %s" % (addr, self.port))
            simple_server_loop(self.drivers, self.bind_address, self.port)
        except (KeyboardInterrupt, SystemExit):
            sys.exit(0)
        finally:
            self.close_drivers()

    def close_drivers(self):
        for device, driver in self.drivers.items():
            if callable(getattr(driver, "close", None)):
                self.logger.info(" Closing driver for %s" % device)
                driver.close()

def camel_case(string):
    return string.title().replace("_", "")


"""
:Created: 13 May 2016
:Author: Lucas Connors

"""

from django.shortcuts import render_to_response

from accounts.models import UserAvatar, UserAvatarURL

def require_email(strategy, details, user=None, is_new=False, *args, **kwargs):
    if not details.get('email'):
        return render_to_response('registration/error_email_required.html')

def save_avatar(strategy, details, user=None, is_new=False, *args, **kwargs):
    # Skip if we don't have the user yet
    if not user:
        return

    # Get avatar from provider, skip if no avatar
    provider = kwargs['backend'].name
    try:
        if provider == 'google-oauth2':
            avatar = kwargs['response']['image']
            is_default_avatar = avatar['isDefault']
        elif provider == 'facebook':
            avatar = kwargs['response']['picture']['data']
            is_default_avatar = avatar['is_silhouette']
        else:
            return
    except KeyError:
        return

    # Skip if the user just has the default avatar
    if is_default_avatar:
        return

    # Get avatar URL from provider
    try:
        avatar_url = avatar['url']
    except KeyError:
        return

    # For Google, use larger image than default
    if provider == 'google-oauth2':
        avatar_url = avatar_url.replace('?sz=50', '?sz=150')

    # Save avatar URL
    user_avatar, created = UserAvatar.objects.get_or_create(user=user, provider=provider)
    user_avatar_url, _ = UserAvatarURL.objects.update_or_create(avatar=user_avatar, defaults={'url': avatar_url,})

    # Update user's current avatar if none was ever set
    if created and not user.userprofile.avatar:
        user.userprofile.avatar = user_avatar
        user.userprofile.save()


from django.core import validators
from django.utils.translation import ugettext_lazy as _
from django import forms

from apps.main.models import Phrase, UserSuggestion, Post
from apps.validators import validate_suggestion

class NewPhraseForm(forms.ModelForm):
    phrase = forms.CharField(max_length=1000,
        label=_("Add a new phrase or a single word"),
        validators=[
            validators.RegexValidator(
                r'^[a-zA-Z,.;]+(?:)*$',
                _(u'Invalid word(s)'),
                code='Invalid word(s)')
        ],
    )
    translation = forms.CharField(max_length=1000,
        label=_("Submit translation")
    )

    class Meta:
        model = Phrase

class UserSuggestionForm(forms.ModelForm):
    def __init__(self, *args, **kwargs):
        super(UserSuggestionForm, self).__init__(*args, **kwargs)
        self.fields['title'].validators.append(validate_suggestion)
        self.fields['body'].validators.append(validate_suggestion)

    class Meta:
        model = UserSuggestion
        fields = ['title', 'body']

class NewPostForm(forms.ModelForm):
    class Meta:
        model = Post
        # exclude = ('slug', 'status',)  Don't use exclude, explicit is better than implicit
        fields = ('title', 'body',)

class PostUpdateForm(forms.ModelForm):
    class Meta:
        model = Post
        fields = ('title', 'body',)


import re
import csv
from io import StringIO
from os import path
from functools import wraps
import unicodedata

from flask import (abort, current_app, session, request, redirect, url_for)
from flask_login import current_user

from notifications_utils.template import (
    SMSPreviewTemplate,
    EmailPreviewTemplate,
    LetterPDFLinkTemplate,
)

import pyexcel
import pyexcel.ext.io
import pyexcel.ext.xls
import pyexcel.ext.xlsx
import pyexcel.ext.ods3

class BrowsableItem(object):
    """
    Maps for the template browse-list.
    """

    def __init__(self, item, *args, **kwargs):
        self._item = item
        super(BrowsableItem, self).__init__()

    @property
    def title(self):
        pass

    @property
    def link(self):
        pass

    @property
    def hint(self):
        pass

    @property
    def destructive(self):
        pass

def user_has_permissions(*permissions, admin_override=False, any_=False):
    def wrap(func):
        @wraps(func)
        def wrap_func(*args, **kwargs):
            if current_user and current_user.is_authenticated:
                if current_user.has_permissions(
                    permissions=permissions,
                    admin_override=admin_override,
                    any_=any_
                ):
                    return func(*args, **kwargs)
                else:
                    abort(403)
            else:
                abort(401)
        return wrap_func
    return wrap

def redirect_to_sign_in(f):
    @wraps(f)
    def wrapped(*args, **kwargs):
        if 'user_details' not in session:
            return redirect(url_for('main.sign_in'))
        else:
            return f(*args, **kwargs)
    return wrapped

def get_errors_for_csv(recipients, template_type):

    errors = []

    if recipients.rows_with_bad_recipients:
        number_of_bad_recipients = len(list(recipients.rows_with_bad_recipients))
        if 'sms' == template_type:
            if 1 == number_of_bad_recipients:
                errors.append("fix 1 phone number")
            else:
                errors.append("fix {} phone numbers".format(number_of_bad_recipients))
        elif 'email' == template_type:
            if 1 == number_of_bad_recipients:
                errors.append("fix 1 email address")
            else:
                errors.append("fix {} email addresses".format(number_of_bad_recipients))
        elif 'letter' == template_type:
            if 1 == number_of_bad_recipients:
                errors.append("fix 1 address")
            else:
                errors.append("fix {} addresses".format(number_of_bad_recipients))

    if recipients.rows_with_missing_data:
        number_of_rows_with_missing_data = len(list(recipients.rows_with_missing_data))
        if 1 == number_of_rows_with_missing_data:
            errors.append("enter missing data in 1 row")
        else:
            errors.append("enter missing data in {} rows".format(number_of_rows_with_missing_data))

    return errors

def generate_notifications_csv(json_list):
    from app import format_datetime_24h, format_notification_status
    content = StringIO()
    retval = None
    with content as csvfile:
        csvwriter = csv.writer(csvfile)
        csvwriter.writerow(['Row number', 'Recipient', 'Template', 'Type', 'Job', 'Status', 'Time'])
        for x in json_list:
            csvwriter.writerow([
                int(x['job_row_number']) + 2 if 'job_row_number' in x and x['job_row_number'] else '',
                x['to'],
                x['template']['name'],
                x['template']['template_type'],
                x['job']['original_file_name'] if x['job'] else '',
                format_notification_status(x['status'], x['template']['template_type']),
                format_datetime_24h(x['created_at'])])
        retval = content.getvalue()
    return retval

def get_page_from_request():
    if 'page' in request.args:
        try:
            return int(request.args['page'])
        except ValueError:
            return None
    else:
        return 1

def generate_previous_dict(view, service_id, page, url_args=None):
    return generate_previous_next_dict(view, service_id, page - 1, 'Previous page', url_args or {})

def generate_next_dict(view, service_id, page, url_args=None):
    return generate_previous_next_dict(view, service_id, page + 1, 'Next page', url_args or {})

def generate_previous_next_dict(view, service_id, page, title, url_args):
    return {
        'url': url_for(view, service_id=service_id, page=page, **url_args),
        'title': title,
        'label': 'page {}'.format(page)
    }

def email_safe(string, whitespace='.'):
    # strips accents, diacritics etc
    string = ''.join(c for c in unicodedata.normalize('NFD', string) if unicodedata.category(c) != 'Mn')
    string = ''.join(
        word.lower() if word.isalnum() or word == whitespace else ''
        for word in re.sub(r'\s+', whitespace, string.strip())
    )
    string = re.sub(r'\.{2,}', '.', string)
    return string.strip('.')

class Spreadsheet():

    allowed_file_extensions = ['csv', 'xlsx', 'xls', 'ods', 'xlsm', 'tsv']

    def __init__(self, csv_data, filename=''):
        self.filename = filename
        self.as_csv_data = csv_data
        self.as_dict = {
            'file_name': self.filename,
            'data': self.as_csv_data
        }

    @classmethod
    def can_handle(cls, filename):
        return cls.get_extension(filename) in cls.allowed_file_extensions

    @staticmethod
    def get_extension(filename):
        return path.splitext(filename)[1].lower().lstrip('.')

    @staticmethod
    def normalise_newlines(file_content):
        return '\r\n'.join(file_content.read().decode('utf-8').splitlines())

    @classmethod
    def from_rows(cls, rows, filename=''):
        with StringIO() as converted:
            output = csv.writer(converted)

            for row in rows:
                output.writerow(row)
            return cls(converted.getvalue(), filename)

    @classmethod
    def from_file(cls, file_content, filename=''):
        extension = cls.get_extension(filename)

        if extension == 'csv':
            return cls(Spreadsheet.normalise_newlines(file_content), filename)

        if extension == 'tsv':
            file_content = StringIO(Spreadsheet.normalise_newlines(file_content))

        return cls.from_rows(pyexcel.get_sheet(
            file_type=extension,
            file_content=file_content.read()
        ).to_array(), filename)

def get_help_argument():
    return request.args.get('help') if request.args.get('help') in ('1', '2', '3') else None

def is_gov_user(email_address):
    valid_domains = current_app.config['EMAIL_DOMAIN_REGEXES']
    email_regex = (r"[\.|@]({})$".format("|".join(valid_domains)))
    return bool(re.search(email_regex, email_address.lower()))

def get_template(template, service, show_recipient=False, expand_emails=False):
    if 'email' == template['template_type']:
        return EmailPreviewTemplate(
            template,
            from_name=service['name'],
            from_address='{}@notifications.service.gov.uk'.format(service['email_from']),
            expanded=expand_emails,
            show_recipient=show_recipient
        )
    if 'sms' == template['template_type']:
        return SMSPreviewTemplate(
            template,
            prefix=service['name'],
            sender=service['sms_sender'],
            show_recipient=show_recipient
        )
    if 'letter' == template['template_type']:
        return LetterPDFLinkTemplate(
            template,
            service_id=service['id'],
        )


from unityparser import UnityDocument
from enum import Enum
import yaml
import json
import os

def removeUnityTagAlias(filepath):
    """
    Name:               removeUnityTagAlias()
    Description:        Loads a file object from a Unity textual scene file, which is in a pseudo YAML style, and strips the
                        parts that are not YAML 1.1 compliant. Then returns a string as a stream, which can be passed to PyYAML.
                        Essentially removes the "!u!" tag directive, class type and the "&" file ID directive. PyYAML seems to handle
                        rest just fine after that.
    Returns:                String (YAML stream as string)
    """
    result = str()
    sourceFile = open(filepath, 'r')

    fileID = ''
    for lineNumber,line in enumerate( sourceFile.readlines() ):
        if line.startswith('--- !u!'):
            result += '---' + '\n'
            fileID = line.split(' ')[2].replace('&', '')   # remove the tag, but keep file ID
        else:
            # Just copy the contents...
            result += line
            if fileID != '':
                result += ('  instanceID: ' + fileID + '\n')
                fileID = ''

    sourceFile.close()

    return result

def getPrefabGuid(prefabInstance):
    return prefabInstance['m_SourcePrefab']['guid']

class SupportedPrefabGuid(Enum):
    floor = '25399a8663513438abe48fcceca3e09e'
    floorConcave = '3b68c9bc9bd2648e8852d4d2b19e4673'
    ice = '08c6a9191fe224a179b16832b8d4ff63'
    iceConcave = '311bfe237ebcc4c2085daad068a2412b'
    wall = 'e3a352e5e460640c99199adbd07d7925'
    trap = '9a04efca5190044af8c9cb52b1da8944'
    checkpoint = '57b4cb67c63b54e09ab4f8a32995c2ef'
    lava = 'e1858a08e2538704fbb588752fee4e3e'
    spike = 'c5ca291e14223427ca02c6e1181c1d5e'
    door = 'ac57bdd24b4c040218eb99c8a6b95be9'
    triggerBoard = '6cfeb0107473b4df588127baf9804cd5'

class TriggerAction(Enum):
    move = 'Move'
    resetPosition = 'ResetPosition'

def tryGetValueFromDict(dict, name, defaultValue = None):
    if name in dict:
        return dict[name]
    else:
        return defaultValue

def moveActionHandler(data, actionName):
    params = {
        'x':tryGetValueFromDict(data, actionName + '.arguments.Array.data[0].vector3Argument.x'),
        'y':tryGetValueFromDict(data, actionName + '.arguments.Array.data[0].vector3Argument.y'),
        'z':tryGetValueFromDict(data, actionName + '.arguments.Array.data[0].vector3Argument.z')
    }
    return params

class EventName(Enum):
    onPlayerEnterEvent = 'OnPlayerEnterEvent'
    onPlayerExitEvent = 'OnPlayerExitEvent'

numberOfPrefabs = {}
for item in SupportedPrefabGuid:
    numberOfPrefabs[item] = 0
numberOfPrefabs['other'] = 0

levelName = 'All_Levels(Draft 1).unity'

document = os.getcwd() + '/Assets/Scenes/' + levelName
UnityStreamNoTags = removeUnityTagAlias(document)

ListOfNodes = list()

for data in yaml.safe_load_all(UnityStreamNoTags):
    ListOfNodes.append( data )

prefabInstances = list(filter(lambda x: 'PrefabInstance' in x, ListOfNodes))
gameObjectInstances = list(filter(lambda x: 'GameObject' in x, ListOfNodes))
transformInstances = list(filter(lambda x: 'Transform' in x, ListOfNodes))

def setKey(content, defaultValue=None, *names):
    for name in names:
        if name not in content:
            content[name] = defaultValue

def copyItems(target, source, *names):
    for name in names:
        if source and name in source:
            target[name] = source[name]
        else:
            target[name] = None

def findNodeByID(instanceID, content=ListOfNodes):
    for item in content:
        target = list(item.values())[0]
        if target['instanceID'] == instanceID:
            return target
    return None

def eventListHandler(content, sEventName, actionList):
    if sEventName + '.persistentCalls.calls.Array.size' not in content:
        return
    for x in range(content[sEventName + '.persistentCalls.calls.Array.size']):
            actionName = sEventName + '.persistentCalls.calls.Array.data[{}]'.format(x)
            if actionName + '.memberName' in content:
                action = {}
                action['event'] = sEventName
                action['target'] = content[actionName + '.target']
                action['memberName'] = content[actionName + '.memberName']
                if action['memberName'] == TriggerAction.move.value:
                    action['params'] = moveActionHandler(content, actionName)
                elif action['memberName'] == TriggerAction.resetPosition:
                    pass
                actionList.append(action)

tileFilterKeys = ('m_InstanceID', 'm_Name', 'm_IsActive', 'm_LocalPosition', 'm_LocalRotation',
     'm_SourcePrefab', 'm_Attributes', 'm_Actions')

maps = {}
for node in prefabInstances:
    dataDict = {}
    result = {}
    municipality = False
    levelTransformID = node['PrefabInstance']['m_Modification']['m_TransformParent']['fileID']
    levelTransform = findNodeByID(levelTransformID, transformInstances)
    levelGameObject = mapTransform = mapGameObject = None
    if not levelTransform:
        municipality = True
    else:
        if 'm_GameObject' not in levelTransform:
            continue
        levelGameObject = findNodeByID(levelTransform['m_GameObject']['fileID'], gameObjectInstances)
        mapTransform = findNodeByID(levelTransform['m_Father']['fileID'], transformInstances)
        if not mapTransform:
            municipality = True
        else:
            if 'm_GameObject' not in mapTransform:
                continue
            mapGameObject = findNodeByID(mapTransform['m_GameObject']['fileID'], gameObjectInstances)
    if not municipality:
        mapNode = levelNode = None
        if mapGameObject['m_Name'] not in maps:
            maps[mapGameObject['m_Name']] = {}
            mapNode = maps[mapGameObject['m_Name']]
            mapNode['m_InstanceID'] = mapTransform['instanceID']
            copyItems(mapNode, mapGameObject, 'm_Name', 'm_IsActive')
            copyItems(mapNode, mapTransform, 'm_LocalPosition', 'm_LocalRotation')
            mapNode['m_Levels'] = {}
            mapNode['m_Levels'][levelGameObject['m_Name']] = {}
        elif levelGameObject['m_Name'] not in maps[mapGameObject['m_Name']]['m_Levels']:
            maps[mapGameObject['m_Name']]['m_Levels'][levelGameObject['m_Name']] = {}
        mapNode = maps[mapGameObject['m_Name']]
        levelNode = mapNode['m_Levels'][levelGameObject['m_Name']]
        levelNode['m_InstanceID'] = levelTransform['instanceID']
        copyItems(levelNode, levelGameObject, 'm_Name', 'm_IsActive')
        copyItems(levelNode, levelTransform,'m_LocalPosition', 'm_LocalRotation')
        if 'm_Tiles' not in levelNode:
            levelNode['m_Tiles'] = []
        levelNode['m_Tiles'].append(result)

    modifications = node['PrefabInstance']['m_Modification']['m_Modifications']
    for data in modifications:
        if data['value'] != None:
            dataDict[data['propertyPath']] = data['value']
        else:
            if data['objectReference']['fileID'] == 0:
                continue
            dataDict[data['propertyPath']] = findNodeByID(data['objectReference']['fileID'])['m_PrefabInstance']['fileID']
    dataDict['m_SourcePrefab'] = getPrefabGuid(node['PrefabInstance'])
    dataDict['m_InstanceID'] = node['PrefabInstance']['instanceID']
    dataDict['m_LocalPosition'] = {'x': dataDict['m_LocalPosition.x'], 'y': dataDict['m_LocalPosition.y'], 'z': dataDict['m_LocalPosition.z']}
    dataDict['m_LocalRotation'] = {'x': dataDict['m_LocalRotation.x'], 'y': dataDict['m_LocalRotation.y'],
        'z': dataDict['m_LocalRotation.z'], 'w': dataDict['m_LocalRotation.w']}
    dataDict['m_Attributes'] = {}
    setKey(dataDict, 1, 'm_IsActive')
    # floor logic
    if dataDict['m_SourcePrefab'] == SupportedPrefabGuid.floor.value:
        numberOfPrefabs[SupportedPrefabGuid.floor] = numberOfPrefabs[SupportedPrefabGuid.floor] + 1
        setKey(dataDict['m_Attributes'], None, 'm_floorState', 'declineAfterExit', 'stepsBeforeIncline')
    # floorConcave logic
    elif dataDict['m_SourcePrefab'] == SupportedPrefabGuid.floorConcave.value:
        numberOfPrefabs[SupportedPrefabGuid.floorConcave] = numberOfPrefabs[SupportedPrefabGuid.floorConcave] + 1
    # ice logic
    elif dataDict['m_SourcePrefab'] == SupportedPrefabGuid.ice.value:
        numberOfPrefabs[SupportedPrefabGuid.ice] = numberOfPrefabs[SupportedPrefabGuid.ice] + 1
    # iceConcave logic
    elif dataDict['m_SourcePrefab'] == SupportedPrefabGuid.iceConcave.value:
        numberOfPrefabs[SupportedPrefabGuid.iceConcave] = numberOfPrefabs[SupportedPrefabGuid.iceConcave] + 1
    # wall logic
    elif dataDict['m_SourcePrefab'] == SupportedPrefabGuid.wall.value:
        numberOfPrefabs[SupportedPrefabGuid.wall] = numberOfPrefabs[SupportedPrefabGuid.wall] + 1
    # trap logic
    elif dataDict['m_SourcePrefab'] == SupportedPrefabGuid.trap.value:
        numberOfPrefabs[SupportedPrefabGuid.trap] = numberOfPrefabs[SupportedPrefabGuid.trap] + 1
    # checkpoint logic
    elif dataDict['m_SourcePrefab'] == SupportedPrefabGuid.checkpoint.value:
        numberOfPrefabs[SupportedPrefabGuid.checkpoint] = numberOfPrefabs[SupportedPrefabGuid.checkpoint] + 1
        setKey(dataDict['m_Attributes'], None, 'respawnObject')
    # lava logic
    elif dataDict['m_SourcePrefab'] == SupportedPrefabGuid.lava.value:
        numberOfPrefabs[SupportedPrefabGuid.lava] = numberOfPrefabs[SupportedPrefabGuid.lava] + 1
    # spike logic
    elif dataDict['m_SourcePrefab'] == SupportedPrefabGuid.spike.value:
        numberOfPrefabs[SupportedPrefabGuid.spike] = numberOfPrefabs[SupportedPrefabGuid.spike] + 1
        setKey(dataDict['m_Attributes'], None, 'spikeUp')
    # door logic
    elif dataDict['m_SourcePrefab'] == SupportedPrefabGuid.door.value:
        numberOfPrefabs[SupportedPrefabGuid.door] = numberOfPrefabs[SupportedPrefabGuid.door] + 1
    # triggerBoard logic
    elif dataDict['m_SourcePrefab'] == SupportedPrefabGuid.triggerBoard.value:
        numberOfPrefabs[SupportedPrefabGuid.triggerBoard] = numberOfPrefabs[SupportedPrefabGuid.triggerBoard] + 1
        actionList = list()
        eventListHandler(dataDict, EventName.onPlayerEnterEvent.value, actionList)
        eventListHandler(dataDict, EventName.onPlayerExitEvent.value, actionList)
        setKey(dataDict['m_Attributes'], actionList, 'm_Actions')
    # other logic
    else:
        numberOfPrefabs['other'] = numberOfPrefabs['other'] + 1
    for k in tileFilterKeys:
        if k in dataDict:
            result[k] = dataDict[k]
    #print(result)

sceneData = []
for k, v in maps.items():
    mapData = []
    for key, value in v['m_Levels'].items():
        mapData.append(value)
    v['m_Levels'] = mapData
    sceneData.append(v)

outfilePath = os.getcwd() + '/Assets/StreamingAssets/' + levelName + '.json'
outfile = open(outfilePath, 'w')
json.dump(sceneData, outfile, indent=4)
print('Successfully export to ' + outfilePath + '!')
outfile.close()


import time
import os
import ast
from robot.api import logger
from robot.api.deco import keyword
from robot.libraries.BuiltIn import BuiltIn
from robot.libraries.BuiltIn import RobotNotRunningError
from platina_sdk import pcc_api as pcc
from aa.common import PccUtility as easy
from aa.common.Utils import banner, trace, pretty_print
from aa.common.Result import get_response_data, get_result
from aa.common.AaBase import AaBase

class NodeRoles(AaBase):
    """
    NodeRoles
    """

    def __init__(self):
        self.Name = None
        self.nodes = None
        self.Id = None
        self.Description = None
        self.owners = []
        self.templateIDs = []
        self.number_of_node_roles = None
        self.templateNames = None
        super().__init__()

    ###########################################################################
    @keyword(name="PCC.Get Node Roles")
    ###########################################################################
    def get_node_roles(self, *args, **kwargs):
        """
        Get Node Roles from PCC
        [Args]
            None
        [Returns]
            (dict) Response: Get Node Roles response (includes any errors)
        """
        banner("PCC.Get Node Roles")
        self._load_kwargs(kwargs)
        conn = BuiltIn().get_variable_value("${PCC_CONN}")
        return pcc.get_roles(conn)

    ###########################################################################
    @keyword(name="PCC.Add Node Role")
    ###########################################################################
    def add_node_role(self, *args, **kwargs):
        """
        Add Node Role
        [Args]
            (str) name: name of the Node Role
        [Returns]
            (dict) Response: Add Node Role response (includes any errors)
        """
        self._load_kwargs(kwargs)
        banner("PCC.Add Node Role [Name=%s]" % self.Name)
        conn = BuiltIn().get_variable_value("${PCC_CONN}")
        print("Kwargs are: {}".format(kwargs))
        if "Description" not in kwargs:
            self.Description = None

        payload ={
                  "name":self.Name,
                  "description":self.Description,
                  "templateIDs":ast.literal_eval(self.templateIDs),
                  "owners":ast.literal_eval(self.owners)
                  }

        return pcc.add_role(conn, data=payload)

    ###########################################################################
    @keyword(name="PCC.Get Template Id")
    ###########################################################################
    def get_template_id_by_name(self, *args, **kwargs):
        """
        Get Id of Application with matching Name
        [Args]
            (dict) conn: Connection dictionary obtained after logging in
            (str) Name: Name of the Template
        [Returns]
            (int) Id: Id of the matchining Application, or
                None: if no match found, or
            (dict) Error response: If Exception occured
        """
        self._load_kwargs(kwargs)
        banner("PCC.Get Template Id [Name=%s]" % self.Name)
        conn = BuiltIn().get_variable_value("${PCC_CONN}")
        print("Kwargs are: {}".format(kwargs))

        template_list = pcc.get_templates(conn)['Result']['Data']
        try:
            for template in template_list:
                if str(template['name'].lower()) == str(self.Name).lower():
                    return template['id']
            return None
        except Exception as e:
            return {"Error": str(e)}

    ###########################################################################
    @keyword(name="PCC.Get Node Role Id")
    ###########################################################################
    def get_node_role_id(self, *args, **kwargs):
        """
        Get Node Role Id
        [Args]
            (str) Name
        [Returns]
            (int) Id: Node Role Id if there is one,
                None: If not found
        """
        self._load_kwargs(kwargs)
        banner("PCC.Get Node Role Id [Name=%s]" % self.Name)
        conn = BuiltIn().get_variable_value("${PCC_CONN}")
        return easy.get_node_role_id_by_name(conn=conn, Name=self.Name)

    ###########################################################################
    @keyword(name="PCC.Validate Node Role")
    ###########################################################################
    def validate_node_role_by_name(self, *args, **kwargs):
        """
        Validate Node Role by Name
        [Args]
            (dict) conn: Connection dictionary obtained after logging in
            (str) Name: Name of the Node Role
        [Returns]
            "OK": If node role present in PCC
            else: "Node role not available" : If node role not present in PCC

        """
        self._load_kwargs(kwargs)
        banner("PCC.Validate Node Role")
        conn = BuiltIn().get_variable_value("${PCC_CONN}")
        noderoles_list = pcc.get_roles(conn)['Result']['Data']
        try:
            for noderole in noderoles_list:
                if str(noderole['name']) == str(self.Name):
                    return "OK"
            return "Node role not available"
        except Exception as e:
            return {"Error": str(e)}

    ###########################################################################
    @keyword(name="PCC.Verify Node Role On Nodes")
    ###########################################################################
    def verify_node_roles_on_nodes(self, *args, **kwargs):
        """
        Verify node role on Nodes
        [Args]
            (list) nodes: name of pcc nodes
            (list) roles: name of roles
        [Returns]
            (dict) Response: OK if node role exists on the node (includes any errors)
        """
        self._load_kwargs(kwargs)
        print("kwargs:-"+str(kwargs))
        banner("PCC.Verify Node Role On Nodes")

        conn = BuiltIn().get_variable_value("${PCC_CONN}")
        node_role_id = self.get_node_role_id(conn,Name= self.Name)
        for node in ast.literal_eval(self.nodes):
            print("Node from user is: {}".format(node))
            response = pcc.get_nodes(conn)
            for data in get_response_data(response):
                self.Id=data['Id']
                self.Host=data['Host']
                print("node name from pcc: {}".format(data['Name']).lower())
                if str(data['Name']).lower() == str(node).lower():
                    if data['roles'] == None:
                        return "No roles present on node"
                    if node_role_id in data['roles']:
                        return "OK"
                    else:
                        return "Node role {} not present on node: {}".format(self.Name, node)

    ###########################################################################
    @keyword(name="PCC.Delete Node Role")
    ###########################################################################
    def delete_node_role_by_name(self, *args, **kwargs):
        """
        Delete Node Role by Name
        [Args]
            (dict) conn: Connection dictionary obtained after logging in
            (str) Name
        [Returns]
            (dict) Response: Delete Roles response (includes any errors)
        """
        self._load_kwargs(kwargs)
        banner("PCC.Validate Node Role")
        conn = BuiltIn().get_variable_value("${PCC_CONN}")
        Id = self.get_node_role_id(Name=self.Name)
        return pcc.delete_role_by_id(conn, str(Id))

    ###########################################################################
    @keyword(name="PCC.Modify Node Role")
    ###########################################################################
    def modify_node_role(self, *args, **kwargs):
        """
        Modify Node Role
        [Args]
            (str) name: name of the Node Role
        [Returns]
            (dict) Response: Add Node Role response (includes any errors)
        """
        self._load_kwargs(kwargs)
        banner("PCC.Modify Node Role [Name=%s]" % self.Name)
        conn = BuiltIn().get_variable_value("${PCC_CONN}")
        print("Kwargs are: {}".format(kwargs))

        banner("Type of Id: {}".format(type(kwargs['Id'])))
        if "Description" not in kwargs:
            self.Description = None

        payload ={
                  "id":self.Id,
                  "name":self.Name,
                  "description":self.Description,
                  "templateIDs":ast.literal_eval(self.templateIDs),
                  "owners":ast.literal_eval(self.owners)
                  }

        return pcc.modify_role(conn,str(self.Id), data=payload)

    ###########################################################################
    @keyword(name="PCC.Add Multiple Node Roles")
    ###########################################################################
    def add_multiple_node_roles(self, *args, **kwargs):
        """
        Add Node Role to PCC
        [Args]
            (str) Names: Names of the node role
            (int) owner: Tenant Id for the new group
            (str) Descriptions: Descriptions of the node role
            number_of_node_roles: Number of node roles to be added
        [Returns]
            (dict) Response: Add Node Role response (includes any errors)
        """
        self._load_kwargs(kwargs)
        banner("PCC.Add Multiple Node Roles [Name=%s]" % self.Name)
        conn = BuiltIn().get_variable_value("${PCC_CONN}")

        node_role_dict = {}
        for i in range(1,int(self.number_of_node_roles)+1):
            node_role_dict["node_role{}".format(i)]={"Name":"{}{}".format(self.Name,i),"Description":"{}{}".format(self.Description,i)}

        response_status = []
        availability_status = []
        for node in node_role_dict.values():
            payload= {
                      "name": node["Name"],
                      "description": node["Description"],
                      "templateIDs":ast.literal_eval(self.templateIDs),
                      "owners":ast.literal_eval(self.owners)
                     }

            add_node_role_response = pcc.add_role(conn, data=payload)
            print("add_cluster_response is: ",add_node_role_response)

            response_status.append(add_node_role_response['StatusCode'])
            time.sleep(1)
            availabilty_response = self.validate_node_role_by_name(Name=node["Name"])
            print("availabilty_response is: ",availabilty_response)

            availability_status.append(availabilty_response)

        response_result = len(response_status) > 0 and all(elem == 200 for elem in response_status)
        availability_result = len(availability_status) > 0 and all(elem == availability_status[0] for elem in availability_status)

        if (response_result) and (availability_result):
            return "OK"

        return "Error: All Node Roles not added to PCC"

    ###########################################################################
    @keyword(name="PCC.Delete Multiple Node Roles")
    ###########################################################################
    def delete_multiple_node_roles(self, *args, **kwargs):
        """
        Delete Multiple Node Roles from PCC
        [Args]
            (str) Id: Id of the node role
            number_of_node_roles: Number of node roles to be deleted
        [Returns]
            (dict) Response: Add Node Role response (includes any errors)
        """
        self._load_kwargs(kwargs)
        banner("PCC.Delete Multiple Node Roles [Name=%s]" % self.Name)
        conn = BuiltIn().get_variable_value("${PCC_CONN}")

        node_role_dict = {}

        for i in range(1,int(self.number_of_node_roles)+1):
            node_role_dict["node_role{}".format(i)]={"Name":"{}{}".format(self.Name,i)}

        response_status = []
        availability_status = []
        for node in node_role_dict.values():
            deletion_response = self.delete_node_role_by_name(Name=node["Name"])
            print("deletion_response is: ",deletion_response)

            response_status.append(deletion_response["StatusCode"])
            time.sleep(1)
            availabilty_response = self.validate_node_role_by_name(Name=node["Name"])
            print("availabilty_response is: ",availabilty_response)

            availability_status.append(availabilty_response)

        response_result = len(response_status) > 0 and all(elem == 200 for elem in response_status)
        availability_result = len(availability_status) > 0 and all(elem == availability_status[0] for elem in availability_status)

        if (response_result) and (availability_result):
            return "OK"

        return "Error: All Node Roles not deleted from PCC"

    ###########################################################################
    @keyword(name="PCC.Delete all Node roles")
    ###########################################################################
    def delete_all_node_roles(self, *args, **kwargs):
        """
        Delete all Node Roles
        [Args]
            (dict) conn: Connection dictionary obtained after logging in

        [Returns]
            "OK": If all node roles are deleted
            else "Error"

        """
        banner("Delete All Node roles")
        self._load_kwargs(kwargs)
        conn = BuiltIn().get_variable_value("${PCC_CONN}")
        try:
            response = self.get_node_roles()
            list_node_roles = []

            for node_role in get_response_data(response):
                if node_role['name']== "Default" or node_role['name']== "Baremetal Management Node" or node_role['name']== "Cluster Head" or node_role['name']== "Ceph Resource" or node_role['name']== "Kubernetes Resource" or node_role['name']== "Network Resource":
                    continue
                list_node_roles.append(node_role['name'])
            print("list of node roles: {}".format(list_node_roles))
            response_status = []

            try:
                if list_node_roles == []:
                    return "OK"
                else:
                    for node in list_node_roles:
                        print("Node is : " + node)
                        Id = self.get_node_role_id(Name=node)
                        response = pcc.delete_role_by_id(conn, str(Id))
                        print("Response: {}".format(response))
                        response_status.append(response["StatusCode"])
                    response_result = len(response_status) > 0 and all(elem == 200 for elem in response_status)
                    if response_result:
                        return "OK"
                    else:
                        return response_status
            except Exception as e:
                return {"Error":str(e)}
        except Exception as e:
            logger.console("Error in delete_all_node_roles status: {}".format(e))


import os
import json
import music21 as mc
import numpy as np
from collections import Counter

import seaborn as sns
sns.set()

import random
random.seed(20201308)

from gensim.models import Word2Vec

def get_duration_to_id(X_raw, ONEHOT_DURATION_COUNT):
    x_durations = []
    for x in X_raw:
        for xx in x:
            x_durations.append(xx.split('_')[-1])
    print("Found", len(set(x_durations)), "unique durations")
    most_common_d = sorted([pair[0] for pair in Counter(x_durations).most_common(ONEHOT_DURATION_COUNT)])
    d_to_id = {d : idx for idx, d in enumerate(most_common_d)}
    return d_to_id

def _encode_duration(d, duration_to_id):
    onehot = np.zeros(len(duration_to_id)+1)
    d_str = str(d)
    try:
        d_idx = duration_to_id[d_str]
    except KeyError: #outside
        d_idx = len(duration_to_id)
    onehot[d_idx] = 1.0
    return onehot

def get_parsed_data(data_path):
    with open(os.path.join(data_path, "parse_4.json"), "r") as handle:
        return json.load(handle)

def _fix_with_most_common(chords_mapping_dict):
    chords_mapping_dict['F major seventh chord'] = 'C+E+F+A'
    chords_mapping_dict['C major second major tetrachord'] = 'C+D+E+G'
    chords_mapping_dict['enharmonic equivalent to major triad above C#'] = 'C#+F+G#'
    chords_mapping_dict['C major seventh chord'] = 'C+E+G+B'
    chords_mapping_dict['Bb major seventh chord'] = 'D+F+A+B-'
    chords_mapping_dict['F# half diminished seventh chord'] = 'C+E+F#+A'
    chords_mapping_dict['F phrygian tetrachord'] = 'E+F+G+A'
    chords_mapping_dict['C# quartal tetramirror'] = 'C#+E+F#+B'
    chords_mapping_dict['B tritone fourth'] = 'C+F+B'
    chords_mapping_dict['C minor seventh chord'] = 'C+E-+F+G#'
    chords_mapping_dict['Eb quartal trichord'] = 'E-+F+B-'
    chords_mapping_dict['enharmonic equivalent to minor triad above Eb'] = 'E-+G#+B'
    chords_mapping_dict['Eb major seventh chord'] = 'D+E-+G+B-'
    chords_mapping_dict['C# minor seventh chord'] = 'C#+E+G#+B'
    chords_mapping_dict['Eb phrygian tetrachord'] = 'D+E-+F+G'
    chords_mapping_dict['C# major second major tetrachord'] = 'C#+E-+F+G#'
    chords_mapping_dict['E tritone fourth'] = 'E+F+B-'
    chords_mapping_dict['enharmonic to dominant seventh chord above C#'] = 'C#+E-+G+B-'
    chords_mapping_dict['A tritone fourth'] = 'E-+E+A'
    chords_mapping_dict['C# half diminished seventh chord'] = 'C#+E+G+B'
    chords_mapping_dict['E all interval tetrachord'] = 'E+F+G+B'
    chords_mapping_dict['C phrygian tetrachord'] = 'C+D+E+B'
    chords_mapping_dict['E whole tone tetramirror'] = 'D+E+F#+G#'
    chords_mapping_dict['A all interval tetrachord'] = 'C#+E-+E+A'
    chords_mapping_dict['G# minor second diminished tetrachord'] = 'D+G#+A+B'
    chords_mapping_dict['C perfect fourth major tetrachord'] = 'C+E+F+G'
    chords_mapping_dict['G# whole tone tetramirror'] = 'C+D+G#+B-'
    chords_mapping_dict['enharmonic equivalent to diminished triad above C#'] = 'C#+E+B-'
    chords_mapping_dict['C all interval tetrachord'] = 'C+D+E-+G#'
    chords_mapping_dict['C# minor tetramirror'] = 'C#+G#+B-+B'
    chords_mapping_dict['A double fourth tetramirror'] = 'C#+D+G#+A'
    chords_mapping_dict['D all interval tetrachord'] = 'D+E-+F+A'
    chords_mapping_dict['F augmented major tetrachord'] = 'C#+F+G#+A'
    chords_mapping_dict['C minor second quartal tetrachord'] = 'C+D+G+G#'
    chords_mapping_dict['F# minor second diminished tetrachord'] = 'C+F#+G+A'
    chords_mapping_dict['F# tritone fourth'] = 'C+F#+G'
    chords_mapping_dict['C augmented seventh chord'] = 'C+D+E+G#'
    chords_mapping_dict['Eb minor second quartal tetrachord'] = 'D+E-+G+A'
    chords_mapping_dict['Eb perfect fourth major tetrachord'] = 'E-+E+F#+B'
    chords_mapping_dict['Eb tritone fourth'] = 'D+E-+G#'
    chords_mapping_dict['A tritone quartal tetrachord'] = 'C#+D+G+A'
    chords_mapping_dict['F# minor tetramirror'] = 'E+F#+G+A'
    chords_mapping_dict['Eb double fourth tetramirror'] = 'D+E-+G+G#'
    chords_mapping_dict['C# tritone fourth'] = 'C#+D+G'
    chords_mapping_dict['C# perfect fourth minor tetrachord'] = 'C#+E-+F+B-'
    chords_mapping_dict['F augmented seventh chord'] = 'C#+E-+F+A'
    chords_mapping_dict['Bb all interval tetrachord'] = 'D+E+F+B-'
    chords_mapping_dict['Bb augmented major tetrachord'] = 'D+F#+A+B-'
    chords_mapping_dict['C# minor second diminished tetrachord'] = 'C#+E+B-+B'
    return chords_mapping_dict

def get_chords_mapping(data_path):
    with open(os.path.join(data_path, "chord_names.json"), "r") as handle:
        content = json.load(handle)
    new_dict = {content[key] : key for key in content}
    new_dict = _fix_with_most_common(new_dict)
    return new_dict

def name_to_ps(name):
    return mc.pitch.Pitch(name).ps if name != "REST" else -1.0

def unify_note(note):
    name, duration = note.split('_')
    pitch = name_to_ps(name)
    return f"{pitch}_{duration}"

chord_names_dict = {}

def fix_equivalent_name(name):
    part2, part1 = name.split('enharmonic equivalent to ')[-1].split(' above ')
    return ' '.join([part1, part2])

def unify_chord(chord, truncate_chord=4):
    names, duration = chord.split('_')
    new_names = []

    for name in names.split('.'):
        if name not in new_names:
            new_names.append(name)
    new_names = new_names[:truncate_chord]

    new_name = '+'.join(new_names)
    global chord_names_dict
    if new_name not in chord_names_dict:
        pitched_common_name = mc.chord.Chord(new_names).pitchedCommonName.replace('-', ' ')
#         if 'equivalent' in pitched_common_name:
#             pitched_common_name = fix_equivalent_name(pitched_common_name)
        chord_names_dict[new_name] = pitched_common_name

    return f"{chord_names_dict[new_name]}_{duration}"

def select_test(parsed_data, data_path, chorus_verse_only=True):
    if os.path.isfile(os.path.join(data_path, "test_idx.json")):
        with open(os.path.join(data_path, "test_idx.json"), "r") as handle:
            TEST_SELECT = json.load(handle)
            print("Loaded TEST indices")
    else:
        TEST_SIZE = int(len(parsed_data['parse_success']) * 0.1)
        TEST_SELECT = random.sample(range(0, len(parsed_data['parse_success'])), TEST_SIZE)

        with open(os.path.join(PATH, "test_idx.json"), "w") as handle:
            json.dump(TEST_SELECT, handle)

    result = []
    for idx in TEST_SELECT:
        path = parsed_data["parse_success"][idx]
        if (
            chorus_verse_only and ("chorus" in path.lower() or "verse" in path.lower())
        ) or not chorus_verse_only:
            result.append(idx)

    return result

def select_train(parsed_data, artist="*", test_select=None, chorus_verse_only=True):
    TRAIN_SELECT = []
    for idx, path in enumerate(parsed_data["parse_success"]):
        if (
            (isinstance(artist, str) and (artist == "*" or f"/{artist}/" in path))
            or
            (isinstance(artist, list) and any(f"/{ar}/" in path for ar in artist))
        ) and (idx not in test_select or not test_select):
            if (
                chorus_verse_only and ("chorus" in path.lower() or "verse" in path.lower())
            ) or not chorus_verse_only:
                TRAIN_SELECT.append(idx)
    return TRAIN_SELECT

def select_songs(parsed_data, SELECT, REPEAT=1):
    SONGS = []
    notes_translated, chords_translated = [], []
    for idx, (notes, chords) in enumerate(zip(parsed_data['notes'], parsed_data['chords'])):
        if idx in SELECT:
            notes_translated.append([unify_note(n) for n in notes] * REPEAT)
            chords_translated.append([unify_chord(n) for n in chords] * REPEAT)
            SONGS.append(parsed_data["parse_success"][idx])
    return SONGS, notes_translated, chords_translated

def get_train_songs(parsed_data, TRAIN_SELECT, REPEAT=1):
    TRAIN_SONGS = []
    notes_translated, chords_translated = [], []
    for idx, (notes, chords) in enumerate(zip(parsed_data['notes'], parsed_data['chords'])):
        if idx in TRAIN_SELECT:
            notes_translated.append([unify_note(n) for n in notes] * REPEAT)
            chords_translated.append([unify_chord(n) for n in chords] * REPEAT)
            TRAIN_SONGS.append(parsed_data["parse_success"][idx])
    return TRAIN_SONGS, notes_translated, chords_translated

def get_test_songs(parsed_data, TEST_SELECT, REPEAT=1):
    TEST_SONGS = []
    notes_translated_test, chords_translated_test = [], []
    for idx, (notes, chords) in enumerate(zip(parsed_data['notes'], parsed_data['chords'])):
        if idx in TEST_SELECT:
            notes_translated_test.append([unify_note(n) for n in notes])
            chords_translated_test.append([unify_chord(n) for n in chords])
            TEST_SONGS.append(parsed_data["parse_success"][idx])
    return TEST_SONGS, notes_translated_test, chords_translated_test

def load_embeddings(data_path, embeddings_dir="embeddings", emb_size=100, emb_window=8):
    models = {}
    notes_emb_filename = "embeddings_NOTES_size75_window8_CBOW_H-SFMAX.bin"
    chords_emb_filename = "embeddings_CHORDS-SHORT_size100_window2_CBOW_NEG-S.bin"
    print("> Load notes embeddings:", notes_emb_filename)
    print("> Load chords embeddings:", chords_emb_filename)
    models = {
        "NOTES": Word2Vec.load(os.path.join(data_path, embeddings_dir, "NOTES", notes_emb_filename)),
        "CHORDS": Word2Vec.load(os.path.join(data_path, embeddings_dir, "CHORDS", chords_emb_filename))
    }
    return models

def load_embeddings_OLD(data_path, embeddings_dir="embeddings", emb_size=100, emb_window=8):
    models = {}
    assert emb_size == 100
    assert emb_window == 8

    for emb_type in ["NOTES",]: #, "CHORDS"]:
        emb_filename = f"embeddings_{emb_type}_size{emb_size}_window{emb_window}.bin"
        model = Word2Vec.load(os.path.join(data_path, embeddings_dir, emb_type, emb_filename))
        models[emb_type] = model

    custom_embeddings = "embeddings_CHORDS-SHORT_size100_window4_CBOW_H-SFMAX.bin"
    models["CHORDS"] = Word2Vec.load(os.path.join(data_path, embeddings_dir, "CHORDS", custom_embeddings))

    return models

def decode_note(note):
    pitch, duration = note.split('_')
    return pitch, float(duration)

def align_notes_to_chords(notes, chords):
    note_to_chord_align = []

    chord_idx = 0
    _, current_chord_duration = decode_note(chords[chord_idx])

    offset = 0.0
    for idx, note in enumerate(notes):
        current_align = []
        _, duration = decode_note(note)
        current_align.append(chord_idx)
        offset += duration

        while current_chord_duration < offset:
            if chord_idx == len(chords)-1:
                break
            chord_idx += 1
            _, nd = decode_note(chords[chord_idx])
            current_chord_duration += nd
            current_align.append(chord_idx)

        note_to_chord_align.append(current_align)

    return note_to_chord_align

def extract_notes_pitches_durations(notes_slice):
    input_pitches = []
    input_durations = []
    for n in notes_slice:
        pitch, duration = n.split('_')
        input_pitches.append(float(pitch))
        input_durations.append(float(duration))
    return input_pitches, input_durations

def filter_out_chords(chords, window_size):
    if window_size == 1:
        return [chords[-1]] # only last chord - list for compatibility?
    elif window_size == 2:
        return [chords[0], chords[-1]] # first and last only
    else:
        result = []
        durations = np.array([decode_note(chord)[1] for chord in chords[1:-1]])
        top_durations_arg = np.argpartition(durations, -(window_size-2))[-(window_size-2):]
        top_durations = durations[top_durations_arg]
        for idx, d in enumerate(durations):
            if d in top_durations and len(result) < window_size-2:
                result.append(chords[idx])
            if len(result) >= window_size:
                break

        # always include the first and the last chord
        result = [chords[0], *result, chords[-1]]
        assert len(result) == window_size
        return result

def find_aligned_chords(aligns, target_idx, chords_window=4):
    # this operates on INDICES of chords! this is what 'aligns' store
    target_aligned_chords = aligns[target_idx]
    j = 1
    while len(target_aligned_chords) < chords_window and target_idx-j > 0:
        more_chords = aligns[target_idx-j]
        for ch in more_chords[::-1]:
            if ch not in target_aligned_chords:
                target_aligned_chords.insert(0, ch)
            if len(target_aligned_chords) == chords_window:
                break
        j += 1

    j = 1
    next_chord = None
    while not next_chord and target_idx+j < len(aligns):
        next_note_aligned_chords = aligns[target_idx+j]
        for ch in next_note_aligned_chords:
            if ch != target_aligned_chords[-1]:
                next_chord = ch
                break
        j += 1

    return target_aligned_chords, next_chord

def chord_pitches_to_fixed(pitches, pitches_per_chord=4):
    pitches = pitches.split('+')
    result = [0] * pitches_per_chord
    for i in range(min(pitches_per_chord, len(pitches))):
        result[i] = float(pitches[i])
    return result

def _normalize_octave(note, min_octave=3, max_octave=6):
    if note.octave < min_octave:
        note.octave = min_octave
    if note.octave > max_octave:
        note.octave = max_octave
    return note

def _note_pitched_to_named(note):
    pitched, duration = note.split('_')
    if pitched.replace('#', '').replace('-', '').isalnum():
        # is 'C3'
        name = pitched
    else:
        # probably can convert to float
        pitched = float(pitched)
        if pitched == -1.0:
            name = "REST"
        else:
            mc_note = mc.note.Note(pitched)
            mc_note = _normalize_octave(mc_note)
            name = mc_note.nameWithOctave
    return '_'.join([name, duration])

def _split_notes(notes):
    splitted = [nt.split('_') for nt in notes]
    pitches = [s[0] for s in splitted]
    durations = [s[1] for s in splitted]
    return pitches, durations

def prepare_raw_X_y(input_notes, input_chords, notes_window=8, chords_window=4, use_next_chord=True, rep=1):
    X_notes_raw, X_chords_raw, y_raw = [], [], []
    is_start, row_to_song_idx_map = [], []

    translate_notes = _should_translate_notes(input_notes[-1][-1])

    for s_dx, (ns, chs) in enumerate(zip(input_notes, input_chords)):
        notes = (ns.copy()) * rep
        chords = (chs.copy()) * rep

        note_off = 0.0
        chord_off = 0.0
        chidx = 0
        for i in range(notes_window, len(notes)):
            # ____ NOTES
            notes_slice = notes[i-notes_window:i]
            notes_slice = [_note_pitched_to_named(nt) for nt in notes_slice]
            # ____ TARGET
            target = _note_pitched_to_named(notes[i])

            ds = [float(note.split('_')[-1]) for note in notes_slice]
            if note_off == 0.0:
                note_off += sum(ds)

            while chord_off <= note_off and chidx < len(chords):
                chord_off += float(chords[chidx].split('_')[-1])
                chidx += 1
            # ____ CURRENT CHORD
            current_chord = chords[chidx-1]
            current_chord_rem = chord_off-note_off

            # ____ NEXT_CHORD
            if chidx < len(chords):
                next_chord = chords[chidx]
            else:
                next_chord = 0

            # ____ PREVIOUS CHORD
            prev_chords = chords[max(0, chidx-1-(chords_window-1)):max(0, chidx-1)]
            while len(prev_chords) < chords_window-1:
                prev_chords.insert(0, 0)

            X_notes_raw.append(notes_slice)
            X_chords_raw.append([*prev_chords, (current_chord, current_chord_rem), next_chord])
            y_raw.append(target)

            target_d = float(target.split('_')[-1])
            note_off += target_d

            is_start.append(1 if i == 0 else 0)
            row_to_song_idx_map.append(s_dx)

    assert len(X_notes_raw) == len(X_chords_raw) == len(y_raw) == len(row_to_song_idx_map)
    return X_notes_raw, X_chords_raw, y_raw, is_start, row_to_song_idx_map

def trim_rare_labels(X_notes_raw, X_chords_raw, y_raw, is_start, row_to_song_idx_map, min_freq=3, unk="UNK"):
    y_counts = Counter(y_raw)
    X_notes_new, X_chords_new, y_new, is_start_new, row_to_song_idx_new = [], [], [], [], []

    for xn, xch, y, is_st, s_idx in zip(X_notes_raw, X_chords_raw, y_raw, is_start, row_to_song_idx_map):
        X_notes_new.append(xn)
        X_chords_new.append(xch)

        if y_counts[y] >= min_freq:
            y_new.append(y)
        else:
            y_new.append(unk)

        is_start_new.append(is_st)
        row_to_song_idx_new.append(s_idx)

    return X_notes_new, X_chords_new, y_new, is_start_new, row_to_song_idx_new

def trim_rare_labels_separately(y_pitches_raw, y_durations_raw, min_freq=3, unk="UNK", keep=["START", "END", "0.0"]):
    y_p_counts = Counter(y_pitches_raw)
    y_d_counts = Counter(y_durations_raw)
    y_pitches_new, y_durations_new = [], []

    for y_p, y_d in zip(y_pitches_raw, y_durations_raw):
        if y_p_counts[y_p] >= min_freq or y_p in keep:
            y_pitches_new.append(y_p)
        else:
            y_pitches_new.append(unk)

        if y_d_counts[y_d] >= min_freq or y_d in keep:
            y_durations_new.append(y_d)
        else:
            y_durations_new.append(unk)

    return y_pitches_new, y_durations_new

_note_name_to_pitch_cache = {}
def _note_name_to_pitch(note_name):
    global _note_name_to_pitch_cache
    if note_name == 'REST':
        return -1.0
    else:
        if note_name not in _note_name_to_pitch_cache:
            _note_name_to_pitch_cache[note_name] = mc.note.Note(note_name).pitch.ps
        return _note_name_to_pitch_cache[note_name]

def prepare_input_X(X_notes_raw, X_chords_raw, is_start, row_to_song_idx, mode="", notes_window=8, chords_window=4, chord_size=4, indices_cache={}, embeddings=None, vectorizer=None, chords_mapping={},
duration_to_id_notes=None, duration_to_id_chords=None,):
    if mode.lower() == "embeddings":
        return _prepare_input_X_embeddings(
            X_notes_raw,
            X_chords_raw,
            is_start,
            row_to_song_idx,
            embeddings,
            notes_window=notes_window,
            chords_window=chords_window,
            indices_cache=indices_cache
        )
    elif mode.lower() == "pitches":
        return _prepare_input_X_pitches(
            X_notes_raw,
            X_chords_raw,
            is_start,
            row_to_song_idx,
            notes_window=notes_window,
            chords_window=chords_window,
            indices_cache=indices_cache,
            chord_size=chord_size,
            chords_mapping=chords_mapping
        )
    elif mode.lower() == "custom":
        return _prepare_input_X_custom(
            X_notes_raw,
            X_chords_raw,
            row_to_song_idx,
            embeddings,
            vectorizer,
            duration_to_id_notes,
            duration_to_id_chords,
            notes_window=notes_window,
            chords_window=chords_window,
            indices_cache=indices_cache,
        )

def _neutral_embedding(embeddings):
    return np.mean(embeddings.wv.vectors, axis=0) / np.std(embeddings.wv.vectors, axis=0)

def _chord_emb(chords, ch_embeddings):
    ch_emb = []
    for ch in chords:
        if ch != 0:
            if isinstance(ch, str):
                ch_name = ch.split('_')[0]
            elif isinstance(ch, tuple):
                ch_name = ch[0].split('_')[0]
            try:
                ch_emb.append(ch_embeddings[ch_name])
            except KeyError:
                ch_emb.append(_neutral_embedding(ch_embeddings))
        elif ch == 0:
            ch_emb.append([0] * ch_embeddings.vector_size)
        else:
            raise Exception(f"CHORD is {ch}!")
    return ch_emb

def _encode_embeddings(notes, chords, embeddings, is_start=False, translate_notes=False, notes_window=8, chords_window=3):
    notes_emb_size = embeddings["NOTES"].vector_size
    chords_emb_size = embeddings["CHORDS"].vector_size

    vector = np.zeros(
        notes_window*notes_emb_size + notes_window + (chords_window+1)*chords_emb_size + (chords_window+1) + 1 + 1,
        dtype=np.float32
    )
    of = 0

    # NOTE PITCHES
    n_pitches = [note.split('_')[0] for note in notes]
    if translate_notes:
        n_pitches = [str(_note_name_to_pitch(n)) for n in n_pitches]

    n_emb = []
    for n in n_pitches:
        try:
            n_emb.append(embeddings["NOTES"][n])
        except KeyError:
            n_emb.append(_neutral_embedding(embeddings["NOTES"]))

    vector[of:of+(notes_window*notes_emb_size)] = np.array(n_emb).reshape(-1)
    of += (notes_window*notes_emb_size)

    # NOTE DURATIONS
    vector[of:of+notes_window] = [float(note.split('_')[1]) for note in notes]
    of += notes_window

    # CHORDS PITCHES
    assert len(chords) == chords_window+1
    ch_emb = _chord_emb(chords, embeddings["CHORDS"])
    vector[of:of + (chords_window+1)*chords_emb_size] = np.array(ch_emb).reshape(-1)
    of += (chords_window+1)*chords_emb_size

    # CHORDS DURATIONS
    ch_durations = []
    for ch in chords:
        if ch != 0:
            if isinstance(ch, str):
                d = [float(ch.split('_')[1])]
            elif isinstance(ch, tuple):
                d1 = float(ch[0].split('_')[1]) # full duration
                d2 = ch[1] # currnt duration
                d = [d1, d2]
            ch_durations.extend(d)
        elif ch == 0:
            ch_durations.append(0)
        else:
            raise Exception(f"CHORD is {ch}!")

    vector[of:of + (chords_window+1+1)] = np.array(ch_durations)
    of += (chords_window+1+1)

    # IS START
    vector[-1] = 1 if is_start else 0

    return vector

def _encode_custom(notes, chords, embeddings, vectorizer, duration_to_id_notes, duration_to_id_chords, translate_notes=False, notes_window=8, chords_window=3):
    notes_emb_size = embeddings["NOTES"].vector_size
    vectorizer_size = len(vectorizer.vocabulary_)

    vector = np.zeros(
        notes_window*notes_emb_size + notes_window*(len(duration_to_id_notes)+2) + (chords_window+1)*vectorizer_size + (chords_window+1+1)*(len(duration_to_id_chords)+2),
        dtype=np.float32
    )
    of = 0

    # NOTE PITCHES
    n_pitches = [note.split('_')[0] for note in notes]
    if translate_notes:
        n_pitches = [str(_note_name_to_pitch(n)) for n in n_pitches]

    n_emb = []
    for n in n_pitches:
        try:
            n_emb.append(embeddings["NOTES"][n])
        except KeyError:
            n_emb.append(_neutral_embedding(embeddings["NOTES"]))

    vector[of:of+(notes_window*notes_emb_size)] = np.array(n_emb).reshape(-1)
    of += (notes_window*notes_emb_size)

    # NOTE DURATIONS
    for d in [note.split('_')[1] for note in notes]:
        onehot = _encode_duration(d, duration_to_id_notes)
        vector[of:of+len(onehot)] = onehot
        of += len(onehot)
        vector[of] = float(d)
        of += 1

    # CHORDS PITCHES
    assert len(chords) == chords_window+1
    for ch in chords:
        if isinstance(ch, str):
            emb = vectorizer.transform([ch]).toarray().reshape(-1)
        else:
            emb = [0] * vectorizer_size
        vector[of:of+vectorizer_size] = emb
        of += vectorizer_size

    # CHORDS DURATIONS
    ch_durations = []
    for ch in chords:
        if ch != 0:
            if isinstance(ch, str):
                d = ch.split('_')[1]
                d_vec = [*_encode_duration(d, duration_to_id_chords), float(d)]
            elif isinstance(ch, tuple):
                d1 = ch[0].split('_')[1] # full duration
                d2 = ch[1] # currnt duration
                d_vec = [*_encode_duration(d1, duration_to_id_chords), float(d1), *_encode_duration(d2, duration_to_id_chords), float(d2)]
            ch_durations.extend(d_vec)
        elif ch == 0:
            ch_durations.extend([0] * (len(duration_to_id_chords) + 1 + 1))
        else:
            raise Exception(f"CHORD is {ch}!")

    assert len(ch_durations) == (chords_window+1+1) * (len(duration_to_id_chords)+1+1)
    vector[of:of + len(ch_durations)] = np.array(ch_durations)
    of += len(ch_durations)

    assert of == len(vector)
    return vector

def _prepare_input_X_embeddings(X_notes_raw, X_chords_raw, is_start, row_to_song_idx, embeddings, notes_window, chords_window, indices_cache):
    notes_emb_size = embeddings["NOTES"].vector_size
    chords_emb_size = embeddings["CHORDS"].vector_size

    chords_neutral_emb = _neutral_embedding(embeddings["CHORDS"])
    chord_miss_count = 0

    X = np.zeros(
        (
                len(X_notes_raw),
                                                        #embed each chord   #prev, current & next duration: current x2!
                notes_window*notes_emb_size + notes_window + (chords_window+1)*chords_emb_size + (chords_window+1) + 1 + 1 # + is_start
        ),
        dtype=np.float32
    )

    # NOTE PITCH, NOTE DURATION, CHORD PITCHES, CHORD DURATION, IS_START

    iterator = list(enumerate(zip(X_notes_raw, X_chords_raw, is_start, row_to_song_idx)))

    sample_note = X_notes_raw[-1][-1].split('_')[0]
    translate_notes = _should_translate_notes(X_notes_raw[-1][-1])

    for idx, (notes, chords, st, song_idx) in iterator:
        X[idx, :] = _encode_embeddings(
            notes,
            chords,
            embeddings,
            is_start=False,
            translate_notes=translate_notes,
            notes_window=notes_window,
            chords_window=chords_window,
        )

        # CACHE HELPER
        try:
            indices_cache[song_idx].append(idx)
        except KeyError:
            indices_cache[song_idx] = [idx]

    # shrink cache helper
    for idx in indices_cache:
        indices_cache[idx] = [indices_cache[idx][0], indices_cache[idx][-1]]

    print(f"No. UNK chords: {chord_miss_count}")
    return X, indices_cache

def _should_translate_notes(sample):
    sample_note = sample.split('_')[0].replace('-', '').replace('#', '')
    if sample_note.isalnum() or sample_note == "REST":
        return True
    return False

def _prepare_input_X_custom(
    X_notes_raw,
    X_chords_raw,
    row_to_song_idx,
    embeddings,
    vectorizer,
    duration_to_id_notes,
    duration_to_id_chords,
    notes_window,
    chords_window,
    indices_cache
):
    notes_emb_size = embeddings["NOTES"].vector_size
    chords_neutral_emb = _neutral_embedding(embeddings["CHORDS"])
    vectorizer_size = len(vectorizer.vocabulary_)

    X = np.zeros(
        (
                len(X_notes_raw),
                notes_window*notes_emb_size + notes_window*(len(duration_to_id_notes)+2) + (chords_window+1)*vectorizer_size + (chords_window+1+1)*(len(duration_to_id_chords)+2)
        ),
        dtype=np.float32
    )

    # NOTE PITCH, NOTE DURATION, CHORD PITCHES, CHORD DURATION, IS_START

    iterator = list(enumerate(zip(X_notes_raw, X_chords_raw, row_to_song_idx)))
    translate_notes = _should_translate_notes(X_notes_raw[-1][-1])

    for idx, (notes, chords, song_idx) in iterator:
        X[idx, :] = _encode_custom(
            notes,
            chords,
            embeddings,
            vectorizer,
            duration_to_id_notes,
            duration_to_id_chords,
            translate_notes=translate_notes,
            notes_window=notes_window,
            chords_window=chords_window,
        )

        # CACHE HELPER
        try:
            indices_cache[song_idx].append(idx)
        except KeyError:
            indices_cache[song_idx] = [idx]

    # shrink cache helper
    for idx in indices_cache:
        indices_cache[idx] = [indices_cache[idx][0], indices_cache[idx][-1]]

    return X, indices_cache

def _encode_pitches(notes, chords, is_start=False, translate_notes=False, notes_window=8, chords_window=3, chord_size=4, chords_mapping={}):
    vector = np.zeros(
        (notes_window*1 + notes_window + (chords_window+1)*chord_size + (chords_window+1) + 1 + 1),
        dtype=np.float32
    )
    of = 0

    # NOTE PITCHES
    n_pitches = [note.split('_')[0] for note in notes]
    if translate_notes:
        n_pitches = [_note_name_to_pitch(n) for n in n_pitches]
    else:
        n_pitches = [float(n) for n in n_pitches]

    vector[of:of+(notes_window*1)] = np.array(n_pitches).reshape(-1)
    of += (notes_window*1)

    # NOTE DURATIONS
    n_durations = [float(note.split('_')[1]) for note in notes]
    vector[of:of+notes_window] = n_durations
    of += notes_window

    # CHORDS PITCHES
    assert len(chords) == chords_window+1
    ch_p = []
    for ch in chords:
        if ch != 0:
            if isinstance(ch, str):
                ch_name = ch.split('_')[0]
            elif isinstance(ch, tuple):
                ch_name = ch[0].split('_')[0]
            ch_notes = chords_mapping[ch_name]
            ch_pitches = [_note_name_to_pitch(n) for n in ch_notes.split('+')][:chord_size]
            ch_pitches += [0] * max(0, chord_size-len(ch_pitches))
            assert len(ch_pitches) == chord_size
            ch_p.extend(ch_pitches)
        elif ch == 0:
            ch_p.extend([0] * chord_size)
        else:
            raise Exception(f"CHORD is {ch}!")

    vector[of:of + (chords_window+1)*chord_size] = np.array(ch_p).reshape(-1)
    of += (chords_window+1)*chord_size

    # CHORDS DURATIONS
    ch_durations = []
    for ch in chords:
        if ch != 0:
            if isinstance(ch, str):
                d = [float(ch.split('_')[1])]
            elif isinstance(ch, tuple):
                d1 = float(ch[0].split('_')[1]) # full duration
                d2 = ch[1] # currnt duration
                d = [d1, d2]
            ch_durations.extend(d)
        elif ch == 0:
            ch_durations.append(0)
        else:
            raise Exception(f"CHORD is {ch}!")

    vector[of:of + (chords_window+1+1)] = np.array(ch_durations)
    of += (chords_window+1+1)

    # IS START
    vector[-1] = 1 if is_start else 0
    return vector

def _prepare_input_X_pitches(X_notes_raw, X_chords_raw, is_start, row_to_song_idx, notes_window, chords_window, chord_size, indices_cache, chords_mapping={}):
    X = np.zeros(
        (
                len(X_notes_raw),
                                                        #embed each chord   #prev, current & next duration: current x2!
                notes_window*1 + notes_window + (chords_window+1)*chord_size + (chords_window+1) + 1 + 1 # + is_start
        ),
        dtype=np.float32
    )

    # NOTE PITCH, NOTE DURATION, CHORD PITCHES, CHORD DURATION, IS_START

    iterator = list(enumerate(zip(X_notes_raw, X_chords_raw, is_start, row_to_song_idx)))
    translate_notes = _should_translate_notes(X_notes_raw[-1][-1])

    for idx, (notes, chords, st, song_idx) in iterator:
        vector = _encode_pitches(
            notes,
            chords,
            is_start=st,
            translate_notes=translate_notes,
            notes_window=notes_window,
            chords_window=chords_window,
            chord_size=chord_size,
            chords_mapping=chords_mapping
        )
        X[idx, :] = vector

        # CACHE HELPER
        try:
            indices_cache[song_idx].append(idx)
        except KeyError:
            indices_cache[song_idx] = [idx]

    # shrink cache helper
    for idx in indices_cache:
        indices_cache[idx] = [indices_cache[idx][0], indices_cache[idx][-1]]

    return X, indices_cache

from keras.utils import np_utils

def prepare_input_y(y_raw, limit=None, noneclass=True):
    all_tokens = list(sorted(set(y_raw)))
    if noneclass:
        all_tokens.append('NOTHING')
    token_to_id = {token : idx for idx, token in enumerate(all_tokens)}

    if not limit:
        y = np_utils.to_categorical([token_to_id[token] for token in y_raw], num_classes=len(all_tokens))
    else:
        y = np_utils.to_categorical([token_to_id[token] for token in y_raw[:limit]], num_classes=len(all_tokens))
    return y, token_to_id

def get_duration(piece):
    _, duration = piece.split("_")
    return float(duration)

def chord_to_pitches_duration_fixed(chord):
    pitches, duration = chord.split("_")
    pitches = chord_pitches_to_fixed(pitches)
    return pitches, float(duration)

def extract_X_by_song_idx(X, sample_song_idx, row_to_song_idx_map):
    indices = np.where(np.array(row_to_song_idx_map) == sample_song_idx)[0]
    first = indices[0]
    last = indices[-1]
    return X[first:last+1, :, :]

def extract_Xy_by_song_idx(X, y, sample_song_idx, row_to_song_idx_map, indices_cache={}):
    try:
        indices = indices_cache[sample_song_idx]
    except KeyError:
        indices = np.where(np.array(row_to_song_idx_map) == sample_song_idx)[0]
        indices_cache[sample_song_idx] = indices
    first = indices[0]
    last = indices[-1]
    return X[first:last+1, :, :], y[first:last+1], indices_cache

def extract_first_last_by_song_idx(sample_song_idx, row_to_song_idx_map, indices_cache={}):
    try:
        indices = indices_cache[sample_song_idx]
    except KeyError:
        indices = np.where(np.array(row_to_song_idx_map) == sample_song_idx)[0]
        indices_cache[sample_song_idx] = indices
    first = indices[0]
    last = indices[-1]
    return first, last, indices_cache

def get_bigram_transitions_chord_wise(notes_translated, chords_translated, translate_pitch_to_name=True, flat=False):
    transitions = {}

    for notes, chords in zip(notes_translated, chords_translated):

        note_t, chord_t = 0.0, 0.0
        ch_idx = 0
        prev_note = None
        for note in notes:
            note_tr = _note_pitched_to_named(note) if translate_pitch_to_name else note
            current_chord = chords[ch_idx]
            label = current_chord if not flat else current_chord.split('_')[0]
            if not prev_note:
                transitions.setdefault(label, {}).setdefault("START_0.0", []).append(note_tr)
            else:
                transitions.setdefault(label, {}).setdefault(prev_note, []).append(note_tr)
            note_t += float(note_tr.split('_')[-1])
            while chord_t < note_t:
                chord_t += float(chords[ch_idx].split('_')[-1])
                ch_idx += 1
                if ch_idx == len(chords):
                    break
                else:
                    current_chord = chords[ch_idx]
                    # transitions.setdefault(chords[ch_idx], {}).setdefault(prev_note, []).append(note) # (TO CONSIDER...)

            if ch_idx == len(chords):
                break

        transitions.setdefault(label, {}).setdefault(note_tr, []).append("END_0.0")

    for ch in transitions:
        for prev_note in transitions[ch]:
            data = transitions[ch][prev_note]
            sum_events = len(data)
            counts = Counter(data).most_common()
            new_data = {}
            for pair in counts:
                new_data[pair[0]] = pair[1] / sum_events
            transitions[ch][prev_note] = new_data

    return transitions


# -*- coding: utf-8 -*-

import os
import logging
from util import csvdb
import util.log_util

if __name__ == '__main__':
  file_name = os.path.basename(__file__)[0:-3]
  LOG_FILE_NAME = '%s.log' % file_name
  util.log_util.InitLogging(file_name=LOG_FILE_NAME,
                            file_level=logging.DEBUG,
                            console_level=logging.DEBUG)

  # Constant values
  CSV_FILE_NAME = 'cointegration_group_list.csv'
  HEADER_DICT = {"GroupName":[0, type('')],
                 "Ticker":[1, type('')],
                 "Description":[2, type('')],
                 }
  DSN = 'correl'
  TABLE_NAME = 'cointegration_group'

  csvdb.CsvToDB(CSV_FILE_NAME,
                HEADER_DICT,
                DSN,
                TABLE_NAME,
                2)


import argparse
import csv
import os

def combine_csv(csv_filenames, csv_output, gap=0, normalize=False):

    output_rows = []
    overall_time = 0
    for csv_filename in csv_filenames:
        with open(csv_filename) as csv_file:
            csv_reader = csv.reader(csv_file)
            next(csv_reader) # skip header (time, code, release)
            rows = [map(int, row) for row in csv_reader]

        if rows:
            normalize_time = rows[0][0] if normalize else 0
            cur_time = 0
            for row in rows:
                cur_time = row[0] + overall_time - normalize_time
                row[0] = cur_time
                output_rows.append(','.join(str(x) for x in row) + os.linesep)
            overall_time = cur_time + gap

    # Do this at the end to allow for the same file to be written to as opened
    with open(csv_output, 'w') as output:
        output.write('time,code,release' + os.linesep)
        for row in output_rows:
            output.write(row)

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='Combine or normalize csv key record files')
    parser.add_argument('filenames', type=str, nargs='+', help='csv files to be combine (in order)')
    parser.add_argument('output', type=str, help='output csv file')
    parser.add_argument('--gap', type=int, default=0, help='Millisecond gap between files')
    parser.add_argument('--normalize', default=False, action='store_const', const=True, help='Restart counter at 0 for file(s)')

    args = parser.parse_args()

    combine_csv(args.filenames, args.output, gap=args.gap, normalize=args.normalize)


#!/usr/bin/env python

from tempfile import NamedTemporaryFile

import pytest
import platform

from osxmetadata.attributes import ATTRIBUTES

@pytest.fixture
def temp_file():

    # TESTDIR for temporary files usually defaults to "/tmp",
    # which may not have XATTR support (e.g. tmpfs);
    # manual override here.
    TESTDIR = None
    tempfile = NamedTemporaryFile(dir=TESTDIR)
    tempfilename = tempfile.name
    yield tempfilename
    tempfile.close()

test_data = [
    (attr) for attr in sorted(list(ATTRIBUTES.keys())) if ATTRIBUTES[attr].class_ == str
]
ids = [
    attr for attr in sorted(list(ATTRIBUTES.keys())) if ATTRIBUTES[attr].class_ == str
]

test_data2 = [
    (attribute_name)
    for attribute_name in {
        ATTRIBUTES[attr].name
        for attr in sorted(ATTRIBUTES)
        if ATTRIBUTES[attr].class_ == str
    }
]
ids2 = [
    attribute_name
    for attribute_name in {
        ATTRIBUTES[attr].name
        for attr in sorted(ATTRIBUTES)
        if ATTRIBUTES[attr].class_ == str
    }
]

@pytest.mark.parametrize("attribute", test_data, ids=ids)
def test_str_attribute(temp_file, attribute):
    """ test set_attribute, get_attribute, etc """
    from osxmetadata import OSXMetaData
    from osxmetadata.constants import _FINDER_COMMENT_NAMES

    meta = OSXMetaData(temp_file)
    expected = "Foo"
    meta.set_attribute(attribute, expected)
    got = meta.get_attribute(attribute)
    assert expected == got

    expected += " Bar"
    meta.append_attribute(attribute, " Bar")
    assert meta.get_attribute(attribute) == expected

    meta.clear_attribute(attribute)
    assert meta.get_attribute(attribute) == None

    # test setting empty string
    # setting finder comment to empty string clears it
    # but other fields get set to empty string
    # this mirrors the way finder comments work in mdls
    meta.set_attribute(attribute, "")
    if attribute in _FINDER_COMMENT_NAMES:
        assert meta.get_attribute(attribute) == None
    else:
        assert meta.get_attribute(attribute) == ""

    with pytest.raises(AttributeError):
        meta.update_attribute(attribute, "foo")

    with pytest.raises(TypeError):
        meta.discard_attribute(attribute, "foo")

    with pytest.raises(TypeError):
        meta.remove_attribute(attribute, "foo")

@pytest.mark.parametrize("attribute", test_data2, ids=ids2)
def test_str_attribute_2(temp_file, attribute):
    """ test getting and setting attribute by name """
    from osxmetadata import OSXMetaData

    meta = OSXMetaData(temp_file)
    setattr(meta, attribute, "Foo")
    test_attr = getattr(meta, attribute)
    assert test_attr == "Foo"
    assert meta.get_attribute(attribute) == "Foo"

def test_description(temp_file):
    """ test string functions on one of the str attributes """
    from osxmetadata import OSXMetaData

    meta = OSXMetaData(temp_file)
    meta.description = "Foo"
    assert meta.description == "Foo"
    assert meta.get_attribute("description") == "Foo"

    meta.description += " Bar"
    assert meta.description == "Foo Bar"
    assert meta.get_attribute("description") == "Foo Bar"

    meta.description = None
    assert meta.description == None
    assert meta.get_attribute("description") == None

    meta.description = "Foo"
    assert meta.description == "Foo"
    assert meta.get_attribute("description") == "Foo"


import logging
import library.pynma as pynma
import autosub

log = logging.getLogger('thelogger')

def _send_notify(message):
    nma_instance = pynma.PyNMA(str(autosub.NMAAPI))
    resp = nma_instance.push('Auto-Sub', 'Downloaded a Subtitle', message)
    try:
        if not resp[str(autosub.NMAAPI)][u'code'] == u'200':
            log.error("NMA: Failed to send a notification")
            return False
        else:
            log.info("NMA: Notification sent")
            return True
    except:
        log.error("NMA: Something wrong with API-key, failed")

def test_notify():
    log.debug("NMA: Trying to send a notification")
    message = "Auto-Sub successfully sent a test message"
    return _send_notify(message)

def send_notify(lang, subtitlefile, videofile):
    log.debug("NMA: Trying to send a notification")
    message = "Auto-Sub just downloaded the following subtitle: \n %s" %subtitlefile
    return _send_notify(message)


from math import pi, sin
import numpy as np
import matplotlib.pyplot as plt
from numpy.fft import fft
import matplotlib.patches as mpatches
import symbol as sp
from util_python import datacursor_easy
from MathUtils import *

def decimate(x, M): # s es la primera posicion
    y = []
    values = []

    s = x.xvar[0]
    e = x.xvar[-1] - 1
    #print(s)

    for i in range(-M, s-1, -M):
        y.append(x.yvar[i-s])
        values.append(i)

    y.reverse()
    values.reverse()

    for i in range(0, e+1, M):
        y.append(x.yvar[i-s])
        values.append(i)

    return seq(values, y)

def evaluateDTFT(inputSignal):
    pass

def plot_data():
    fig, ax1 = plt.subplots()
    #patches = []

    n_range = range(-50, 51)

    x1 = seq(n_range, [sin(2*pi * 0.25 * n) for n in n_range])
    y1 = decimate(x1, 4)

    x1dtft = mathUtils.dtft(x1)
    y1dtft = mathUtils.dtft(y1, 4)

    #tx = abs(fft(x)/Nx)# rafa
    # ty = abs(fft(y)/Ny)# rafa
    #
    # ax1.plot(rx, tx, color='blue')
    # ax1.plot(ry, ty, color='red')
    #

    patches = []
    patches.append(mpatches.Patch(color="blue", label="original"))
    patches.append(mpatches.Patch(color="tomato", label="downsampled"))

    plt.legend(handles=patches)
    ax1.plot(x1dtft.xvar, x1dtft.yvar, "blue")
    ax1.plot(y1dtft.xvar, y1dtft.yvar, "tomato")

    plt.xlabel("frecuencia (hz)")
    plt.ylabel("amplitud")

    plt.minorticks_on()
    plt.grid(which='major', linestyle='-', linewidth=0.3, color='black')
    plt.grid(which='minor', linestyle=':', linewidth=0.1, color='black')

    plt.show()

plot_data()


from configuration_global.config_reader import ConfigReader
from configuration_global.logger_factory import LoggerFactory
from keras.preprocessing.image import ImageDataGenerator

class DataProvider():
    def __init__(self):
        self.logger = LoggerFactory()
        self.config = ConfigReader()
        self.dataPath = self.config.training_data_path
        self.testDataPath = self.config.test_data_path

    def get_training_data_set(self):
        train_data_generator = ImageDataGenerator(rescale=1. / 255,
                                                  shear_range=0.2,
                                                  zoom_range=0.2,
                                                  horizontal_flip=True)
        training_set = train_data_generator.flow_from_directory(self.dataPath,
                                                                target_size=(24, 24),
                                                                batch_size=32,
                                                                class_mode='binary')
        return training_set

    def get_test_data_set(self):
        test_data_generator = ImageDataGenerator(rescale=1. / 255)
        test_set = test_data_generator.flow_from_directory(self.testDataPath,
                                                           target_size=(24, 24),
                                                           batch_size=32,
                                                           class_mode='binary')
        return test_set


from django.contrib import admin
from django.urls import path, include

from rest_framework.routers import DefaultRouter
from rest_framework_swagger.views import get_swagger_view
from rest_framework_jwt.views import obtain_jwt_token, verify_jwt_token

from .views import (
  ScreenRoomAPIView,
  TimeRowAPIView,
  InterviewDateAPIView,
  AdminTimeRowListAPIView,
  ScreenTimeRowListAPIView,
  AdminTimeRowDeleteAPIView,
  AdminTimeRowUpdateAPIView,
  AdminDaySchedule,
  WeekScheduleView,
  MonthScheduleView,
  )

from . import views

app_name="api"
schema_view = get_swagger_view(title="SGDID API")

urlpatterns = [
  path('document/',schema_view,name="document"),

  path('token-auth/', obtain_jwt_token),
  path('token-verify/', verify_jwt_token),

  path('interview_date/',InterviewDateAPIView.as_view(),name="date_list"),
  path('timetable/<slug:room_id>/<slug:date_id>/',TimeRowAPIView.as_view(), name="timetable_list"),

  path('admin/timetable/<slug:date_id>/',AdminTimeRowListAPIView.as_view(), name="admin_timetable_list"),
  path('admin/dayschedule/<slug:date_id>/',AdminDaySchedule.as_view(), name="admin_day_schedule"),
  path('admin/weekschedule/<int:year>/<int:weeknumber>/',WeekScheduleView.as_view(), name="admin_week_schedule"),
  path('admin/monthschedule/<int:year>/<int:month>/',MonthScheduleView.as_view(), name="admin_month_schedule"),
  path('admin/timetable/timerow/<int:pk>/',AdminTimeRowDeleteAPIView.as_view(), name="admin_timetable_timerow"),
  path('admin/timetable/timerow/update/<int:pk>/',AdminTimeRowUpdateAPIView.as_view(), name="admin_timetable_update"),

  path('screen/room/',ScreenRoomAPIView.as_view(),name="screen_room_list"),
  path('screen/timetable/<slug:room_id>/',ScreenTimeRowListAPIView.as_view(), name="screen_timetable_list"),

  path('export/xls/day/<slug:date_id>/', views.export_timetable_xls, name='export_timetable_xls'),
  path('export/xls/week/<slug:date_id>/', views.export_timetable_xls_week, name='export_timetable_xls_week'),
  path('export/xls/month/<slug:date_id>/', views.export_timetable_xls_month, name='export_timetable_xls_month'),

]