text:
"""
Created on September 5 2021

This module holds all types of explainers used here: https://towardsdatascience.com/a-deep-dive-into-the-language-of-football-2a2984b6bd21.
It can be easily run as demonstrated in explain.py.

@author: Ofir Magdaci (@Magdaci)

"""

import os
import pickle
import re
from plotly.subplots import make_subplots
import plotly.graph_objects as go
import numpy as np
import pandas as pd
import umap
from gensim.models import Doc2Vec, Word2Vec
from tqdm import tqdm

from lib.data_processing import Corpus
from lib.language_patterns import in_box_scoring_pattern, out_box_scoring_pattern, dribble_pattern, \
    flank_dribble_pattern, \
    forward_pressure_pattern, ANDPattern, ORPattern, _search, \
    is_normal_goal_token, get_tokens_by_regex_pattern, passes_forward_pattern, passes_to_right_pattern, \
    passes_from_right_flank_pattern
from lib.models import plot_embeddings
from lib.params import COLUMNS, PLAYERS, PATHS, SHORTNAMES, DEBUG, ARTIFACTS, MODELS_ARTIFACTS

def estimate_doc_vector(doc2vec_model, doc_: list, steps=10, repetitions=10, norm=True) -> np.array:
    '''
    Infer a given doc_ <repetitions> times and avearaging the results
    :param doc2vec_model: Doc2Vec model to use
    :param doc_: list of tokens/words (indexes) in the langugage
    :param steps: int, number of inference steps to pass to the Doc2Vec model
    :param repetitions: int, number of times to infer the doc_ vector for the mean vector
    :type norm: bool, whether to normalize the output vector or not
    :return: inferred vector (np.array)
    '''
    res = []
    for i in range(repetitions):
        res.append(doc2vec_model.infer_vector(doc_, steps=steps))

    mean_res = np.vstack(res).mean(axis=0)
    if norm:
        return mean_res / np.linalg.norm(mean_res)
    else:
        return mean_res

def create_baseline_player(player_match2vec, players_in_position: list, num_docs_to_use: float, players_corpus) -> (
        np.array, np.array):
    # Sample |S| random matches of players with the matching position_name
    docs_to_ix = {val.lower(): key for key, val in enumerate(players_corpus.documents_names)}
    players_in_position = [val for val in players_in_position if val.lower() in docs_to_ix]
    player_matches_docs = np.random.choice(players_in_position, int(num_docs_to_use))
    player_matches_docs_ix = [docs_to_ix[val] for val in player_matches_docs]

    # Infer vector
    all_baseline_vectors = np.vstack([np.array(player_match2vec[key]) for key in player_matches_docs_ix])
    baseline_vector = all_baseline_vectors.mean(axis=0)

    return baseline_vector, all_baseline_vectors, player_matches_docs_ix

def modify_doc(doc: list, interventions: iter, corpus, limit_interventions=None) -> [list, float, float]:
    '''
    Function receives a doc and modify it by a set of interventions
    :param doc: list of strings
    :param interventions: [{'pattern': regex pattern to identify relevant tokens,
                            'probability': probability to replace pattern,
                            'switch_from': string to replace,
                            'switch_to': string to replace to
    :return: edited doc (list).
    '''
    modified_doc = []
    interventions_count = 0
    modified_token_is_oov_count = 0
    for i, token_ix in enumerate(doc):
        token_was_matched = False
        token_ = corpus.ix_2_token[token_ix]
        for intervention_ in interventions:
            if _search(intervention_['pattern'], token_):
                if np.random.rand() <= intervention_['probability']:
                    # Check limit_interventions
                    if limit_interventions is not None and limit_interventions < interventions_count:
                        break

                    # If token switch occurs:
                    token_was_matched = True
                    modified_token = token_.replace(intervention_['switch_from'], intervention_['switch_to'])
                    if modified_token in corpus.token_2_ix:
                        # The new token is a valid token in the language
                        interventions_count += 1
                        modified_doc.append(corpus.token_2_ix[modified_token])
                        break
                    else:
                        # The new token is out-of-vocabulary
                        modified_token_is_oov_count += 1
                        token_was_matched = False

        # If the token was added so far - add it as-is, without any interventions
        if not token_was_matched:
            modified_doc.append(corpus.token_2_ix[token_])

    return modified_doc, interventions_count, modified_token_is_oov_count

def enrich_doc(doc: list, interventions: iter, players_corpus, limit_interventions=None) -> (list, float):
    '''
    Function receives a doc and adding tokens to according to set of rules
    :param doc: list of strings
    :param interventions: [{'pattern': regex pattern to identify relevant tokens,
                            '<added_action>': name of the action to be added (e.g., '<dribble'>).
                                              Is compared to next_action to validate we do not chain identical actions.
                             'probability': probability to add token,
                             'token_builder': a function that takes relevant token (that matched an intervention)
                                                and return a new token after it
    :return: enriched doc (list).
    '''

    # Iterate
    enriched_doc = []
    intervections_count = 0
    modified_token_is_oov_count = 0
    for i, token_ix in enumerate(doc):
        token_ = players_corpus.ix_2_token[token_ix]
        next_token = players_corpus.ix_2_token[doc[i + 1]] if i + 1 < len(doc) else None
        token_matched = False

        for intervention_ in interventions:
            # Check if pattern match current token
            if _search(intervention_['pattern'], token_):
                # Check if next token is not already the added action (irrelevant if 'skip' activated)
                if next_token is not None and intervention_['<added_action>'] in next_token \
                        and not intervention_.get('skip', False):
                    continue
                # Decide if to append token
                if np.random.rand() <= intervention_['probability']:
                    # Check limit_interventions
                    if limit_interventions is not None and limit_interventions < intervections_count:
                        break

                    intervections_count += 1
                    # Add the current token
                    token_matched = True
                    # Check if intervention means skip the token
                    if intervention_.get('skip', False):
                        break

                    enriched_doc.append(token_ix)

                    # Decide relevant params of the enriched token - e.g., extract proper location
                    token_to_add = intervention_['token_builder'](token_,
                                                                  **intervention_.get('token_builder_args', {}))

                    if token_to_add in players_corpus.token_2_ix:
                        # Add the enriched token
                        enriched_doc.append(players_corpus.token_2_ix[token_to_add])
                        break
                    else:
                        # The new token is out-of-vocabulary
                        modified_token_is_oov_count += 1
                        intervections_count -= 1
                        enriched_doc.pop()
                        token_matched = False

        if not token_matched:
            enriched_doc.append(token_ix)

    return enriched_doc, intervections_count, modified_token_is_oov_count

class ModelExplainer(object):
    pass

class ActionAnalogies(ModelExplainer):
    '''
    ActionAnalogies heavily rely on naming conventions used when building the corpus and the language models.
    However, no error will be raised when deviating these guidelines, and empty output will be returned.
    '''

    def __init__(self, action2vec: Word2Vec, corpus: Corpus, action2vec_embeddings: pd.DataFrame,
                 events_data: pd.DataFrame, **kwargs):
        '''
        :param action2vec: the model to use most_similar() method
        :param action2vec_embeddings: pandas Dataframe of players embeddings (index = player name)
        :param corpus:
        :param kwargs
            - repetitions: number of times to infer docs and average the results, for mitigating randomness
            - k: number of similar tokens to return
        '''

        # Data arguments
        self.min_matches_per_player = kwargs.get('min_matches_per_player', 10)
        self._plot = kwargs.get('plot', False)
        self.verbose = kwargs.get('verbose', False)
        self.model_name = kwargs.get('model_name', 'Player2Vec')
        self.corpus = corpus
        self.action2vec = action2vec
        self.action2vec_embeddings = action2vec_embeddings
        self.vocabulary = self.action2vec_embeddings.index.copy().tolist()
        self.events_data = events_data

        # Analysis arguments
        self.k = kwargs.get('k', 10)
        self.num_examples_per_analogy = kwargs.get('num_examples_per_analogy', 5)

        # supported_skills Dict:
        #   key=skill: str, value=function that return True/False if given token is relevant to skill
        self._supported_tokens_types = {'passes': lambda token: '<pass>' in token,
                                        'in_box_scoring': lambda token: re.match(in_box_scoring_pattern, token),
                                        'out_box_scoring': lambda token: re.match(out_box_scoring_pattern, token),
                                        'dribbling': lambda token: re.match(dribble_pattern, token),
                                        'flank_dribbling': lambda token: re.match(flank_dribble_pattern, token),
                                        'pressure': lambda token: re.match(forward_pressure_pattern, token),
                                        'carry': lambda token: '<carry>' in token,
                                        'offsides': lambda token: '<offside>' in token
                                        }
        self.skills_docs = {}
        self.actions_families = {}
        self.build_tokens_families()

    def get_supported_tokens_families(self):
        return list(self._supported_tokens_types.keys())

    def run_analogy(self, _actions_population: list, _token_modifier_from: str, _token_modifier_to: str,
                    analogy_name='unnamed analogy'):
        '''
        Runs the analogy -
         1. sample token A1 from _actions_population,
         2. modify A1 to A2, by replacing the _token_modifier_from in A with _token_modifier_to\
         3. Sample second token B1, modify it to B2 in the same manner
         4. Find most similar tokens, using action2vec.most_similar to the phrase: A1 - A2 + B2
         Expected answer: B1, or tokens similar to B1
        :param _actions_population: list of language tokens to randomly sample A1, B1
        :param _token_modifier_from: str to change into _token_modifier_to
        :param _token_modifier_to: str to change the _token_modifier_from to
        :param analogy_name: string, name of analysis
        '''
        # Filter actions if they modified version is not in the vocabulary
        actions_population_filtered = [val for val in _actions_population if
                                       val.replace(_token_modifier_from, _token_modifier_to) in self.vocabulary]
        examples_results = []
        if self.verbose:
            print(f"\nAnalogy: {analogy_name}")
        for i in tqdm(range(min(self.num_examples_per_analogy, int(np.floor(len(actions_population_filtered) / 2))))):
            # Get token and find the matching token for the analogy
            A1 = actions_population_filtered[i]
            A2 = A1.replace(_token_modifier_from, _token_modifier_to)
            B1 = actions_population_filtered[-(i + 1)]
            B2 = B1.replace(_token_modifier_from, _token_modifier_to)

            # Transform to normalized vectors
            A1_ix = self.corpus.token_2_ix[A1]
            A2_ix = self.corpus.token_2_ix[A2]
            B2_ix = self.corpus.token_2_ix[B2]

            # king (A1) - queen(A2) = man(B1)-woman(B2) --> woman (B2) + king (A1) - man (B1) = queen(A2)
            # --> positive=['woman', 'king'], negative=['man'] -->
            similar = self.action2vec.most_similar(positive=[A1_ix, B2_ix], negative=[A2_ix])
            if self.verbose:
                print(f' - A1 = {A1} \n - A2 = {A2} \n - B2 =  {B2}')
                print(f" - B1: {self.k} most similar tokens:")
                for sim_ in similar[:self.k]:
                    print(f"   - {self.corpus.ix_2_token[sim_[0]], np.round(sim_[1], 4)}")
                print()
            examples_results.append({'analogy': f' - A1 = {A1} \n - A2 = {A2} \n - B2 =  {B2}',
                                     f'{self.k}_most_similar': similar[:self.k]})

        if self.verbose:
            print('----------------------------------------------------------------')
        return examples_results

    def run_dual_analogy(self, pop_A1, pop_A2, pop_B2, analogy_name=''):
        '''
        This analogy operates between two different tokens, rather getting the second toke by modifying an existing one
        :param pop_A1: words population (list) to sample token A1 from
        :param pop_A2: words population (list) to sample token A2 from
        :param pop_B2: words population (list) to sample token B2 from
        :param analogy_name: string, name of the analogy
        :return: list of most similar words, for each num_examples_per_analogy
        '''
        examples_results = []

        if self.verbose:
            print(f"\nAnalogy: {analogy_name}")
        for i in tqdm(range(self.num_examples_per_analogy)):
            # Get token and find the matching token for the analogy
            A1 = np.random.choice(pop_A1, 1)[0]
            A2 = np.random.choice(pop_A2, 1)[0]
            B2 = np.random.choice(pop_B2, 1)[0]

            # Transform to normalized vecs
            A1_ix = self.corpus.token_2_ix[A1]
            A2_ix = self.corpus.token_2_ix[A2]
            B2_ix = self.corpus.token_2_ix[B2]

            # king-queen = man-woman --> woman + king - man = queen --> positive=['woman', 'king'], negative=['man']
            similar = self.action2vec.most_similar(positive=[A1_ix, B2_ix], negative=[A2_ix])
            if self.verbose:
                print(f' - A1 = {A1} \n - A2 = {A2} \n - B2 =  {B2}')
                print(f" - B1: {self.k} most similar tokens:")
                for sim_ in similar[:self.k]:
                    print(f"   - {self.corpus.ix_2_token[sim_[0]], np.round(sim_[1], 4)}")
            examples_results.append({'analogy': f' - A1 = {A1} \n - A2 = {A2} \n - B2 =  {B2}',
                                     f'{self.k}_most_similar': similar[:self.k]})

        return examples_results

    def build_tokens_families(self):
        self.actions_families['passes'] = [word for word in self.vocabulary if
                                           '<pass>' in word and 'incomplete' not in word
                                           and 'type_na=' not in word and 'aerial_won' not in word]
        self.actions_families['shots'] = [word for word in self.vocabulary if '<shot>' in word]
        self.actions_families['right_foot_shots'] = [word for word in self.actions_families['shots']
                                                     if "body_pa=right" in word]
        self.actions_families['saved_shots'] = [word for word in self.actions_families['shots'] if "=saved" in word]
        inbox_foot_scoring = get_tokens_by_regex_pattern(self.actions_families['shots'], in_box_scoring_pattern)
        self.actions_families['inbox_foot_scoring'] = [token for token in inbox_foot_scoring
                                                       if '|normal|' in token and '_foot' in token and '=goal' in token]
        self.actions_families['short_passes'] = [word for word in self.actions_families['passes'] if "short|" in word]
        self.actions_families['foot_clearances'] = [word for word in self.vocabulary
                                                    if '<clearance>' in word and '_foot' in word]
        self.actions_families['ground_passes'] = [word for word in self.actions_families['passes']
                                                  if "|ground" in word]
        self.actions_families['passes_to_right'] = get_tokens_by_regex_pattern(self.actions_families['passes'],
                                                                               passes_to_right_pattern)
        self.actions_families['passes_forward'] = get_tokens_by_regex_pattern(self.actions_families['passes'],
                                                                              passes_forward_pattern)
        self.actions_families['headers'] = [word for word in self.actions_families['shots'] if "head" in word]
        self.actions_families['passes_from_right_flank'] = get_tokens_by_regex_pattern(self.actions_families['passes'],
                                                                                passes_from_right_flank_pattern)
        self.actions_families['cross_from_right_flank'] = [token for token in self.actions_families['passes_to_right']
                                                           if "|high" in token]
        self.actions_families['ground_passes_from_right_flank'] = [token for token in
                                                                   self.actions_families['passes_from_right_flank']
                                                                   if '|ground' in token]
        self.actions_families['forward_passes'] = get_tokens_by_regex_pattern(self.actions_families['passes'],
                                                                              re.compile(
                                                                                "\(4\/5,3\/5\)\<pass\>\:\( \^ \|"))

    def check_model_support(self, analogy_action_family: str):
        if len(self.actions_families[analogy_action_family]) > 1:
            return True
        else:
            return False

    def actions_analogies_similarities(self, analogies: [dict]):
        '''
        Using Word2Vec most_similar method to find best candidates for model analogies
        Printing analogies results
        '''
        res = {}
        for analogy in analogies:
            analogy_name = analogy.get('analogy_name', str(analogy))
            if self.check_model_support(analogy['action_population']):
                res[analogy_name] = self.run_analogy(self.actions_families[analogy['action_population']],
                                                     analogy['token_modifier_from'], analogy['token_modifier_to'],
                                                     analogy_name=analogy_name)
            else:
                print(f"{analogy['action_population']} is not supported by the given vocabulary")
        return res

    def actions_dual_analogies_similarities(self, analogies: [dict]):
        '''
        Using Word2Vec most_similar method to find best candidates for model analogies
        Printing analogies results
        Here, we expect two populations for tokens to be sampled from
        '''
        res = {}
        for analogy in analogies:
            analogy_name = analogy.get('analogy_name', str(analogy))
            pop_A1_support, pop_A2_support = self.check_model_support(analogy['pop_A1']), \
                                             self.check_model_support(analogy['pop_A2'])
            if pop_A1_support and pop_A2_support:
                res[analogy['analogy_name']] = self.run_dual_analogy(self.actions_families[analogy['pop_A1']],
                                                                     self.actions_families[analogy['pop_A2']],
                                                                     self.actions_families[analogy['pop_B2']],
                                                                     analogy_name=analogy_name)
            else:
                if not pop_A1_support:
                    print(f"{analogy['pop_A1']} is not supported by the given vocabulary")
                if not pop_A2_support:
                    print(f"{analogy['pop_A2']} is not supported by the given vocabulary")
            return res

    def default_run(self):
        results = {'analogies': None, 'dual_analogies': None}
        analogies = [
            dict(action_population='passes_to_right', token_modifier_from="- >", token_modifier_to="- <",
                 analogy_name='Pass direction analogy'),

            dict(action_population='passes_forward', token_modifier_from="^ |", token_modifier_to="v |",
                 analogy_name='Pass direction analogy 2'),

            dict(action_population='passes', token_modifier_from="body_pa=right", token_modifier_to="body_pa=left",
                 analogy_name='Pass foot analogy'),

            dict(action_population='passes', token_modifier_from="body_pa=right_foot", token_modifier_to="body_pa=head",
                 analogy_name='Pass head analogy'),

            dict(action_population='passes', token_modifier_from='|ground', token_modifier_to="|high",
                 analogy_name='Pass height'),

            dict(action_population='short_passes', token_modifier_from='short|', token_modifier_to="long|",
                 analogy_name='Pass distance'),

            dict(action_population='saved_shots', token_modifier_from="=saved", token_modifier_to="=goal",
                 analogy_name='Shot score'),

            dict(action_population='right_foot_shots', token_modifier_from="body_pa=right_foot",
                 token_modifier_to="body_pa=head", analogy_name='Shot body part'),

            dict(action_population='forward_passes', token_modifier_from="4/5", token_modifier_to="3/5",
                 analogy_name='Ball goes forward learning')
        ]

        results['analogies'] = self.actions_analogies_similarities(analogies)

        #  A clearance by foot from ground pass ~ head clearance from cross
        dual_analogies = [
            dict(pop_A1='ground_passes_from_right_flank', pop_A2="foot_clearances", pop_B2="cross_from_right_flank",
                 analogy_name='Header to cross from flank ~ shot from a pass from flank')]
        results['dual_analogies'] = self.actions_dual_analogies_similarities(dual_analogies)

        return results

class PlayersAnalogies(ModelExplainer):
    ''' PlayersAnalogies requires players names to work with, assuming named-based representation. '''

    def __init__(self, player_match2vec: Doc2Vec, players_vectors: pd.DataFrame, corpus: Corpus, **kwargs):

        self.player_match2vec = player_match2vec
        self.corpus = corpus
        self.players_vectors = players_vectors
        self.match_sampling = kwargs.get('match_sampling', 10)
        self.num_similar = kwargs.get('num_similar', 10)
        self.export_artifacts = kwargs.get('export_artifacts', False)
        self.verbose = kwargs.get('verbose', False)

        self.player_to_matches = {
            player_: [doc_ for doc_ in corpus.documents_names if player_ in doc_]
            for player_ in self.players_vectors.index}

        self.docs_to_ix = {val: key for key, val in enumerate(corpus.documents_names)}

    def build_artificial_doc(self):
        pass

    def _players_analogy(self, A1, A2, B2, analogy_name='A1 - A2 + B2 ~ ?'):
        similar_docs_names = []
        similar_docs_values = []
        for i in range(self.match_sampling):
            A1_random_match_ix = self.docs_to_ix[np.random.choice(self.player_to_matches[A1])]
            A1_random_vec = self.player_match2vec[A1_random_match_ix]

            A2_random_match_ix = self.docs_to_ix[np.random.choice(self.player_to_matches[A2])]
            A2_random_vec = self.player_match2vec[A2_random_match_ix]

            B2_random_match_ix = self.docs_to_ix[np.random.choice(self.player_to_matches[B2])]
            B2_random_vec = self.player_match2vec[B2_random_match_ix]

            similar = self.player_match2vec.most_similar(positive=[A1_random_vec, B2_random_vec],
                                                         negative=[A2_random_vec],
                                                         topn=100)
            similar_docs_names.extend([int(item[0]) for item in similar])
            similar_docs_values.extend([item[1] for item in similar])

        mean_similar = pd.DataFrame({'doc_ix': similar_docs_names, 'cosine_similarity': similar_docs_values,
                                     'doc_name': [self.corpus.documents_names[ix] for ix in similar_docs_names]
                                     })
        mean_similar[COLUMNS.PLAYER_NAME] = mean_similar['doc_name'].apply(
            lambda str_: str_.split(self.corpus.separator)[0])
        mean_similar = mean_similar[[COLUMNS.PLAYER_NAME, 'cosine_similarity']].groupby(by=[COLUMNS.PLAYER_NAME]) \
            .agg({'cosine_similarity': [np.size, np.mean]})
        mean_similar.columns = ['count', 'mean']
        mean_similar = mean_similar.loc[mean_similar['count'] >= 5] \
            .sort_values(by=['mean'], ascending=False).head(self.num_similar)
        if self.verbose:
            print('Analogy:', analogy_name)
            print(f' - A1 = {A1} \n - A2 = {A2} \n - B2 =  {B2}')
            print(f" - B1: {self.num_similar} most similar tokens:")
            print(mean_similar)
            print()

        return mean_similar

    def players_analogies_analysis(self, triplets: [dict]):
        '''
        Perform players analogies by the given triplets spcification
        :param triplets: {'A1': <player_name: str, 'A2': <player_name: str, 'B2': <player_name: str,
                        analogy_name: str=''}, ..., }
        :type triplets: list of dicts
        :return: {analogy_name: most_similar pd.DataFrame}
        :rtype: dict
        '''
        res = {}
        for run_config in triplets:
            analogy_name = run_config.get('analogy_name',
                                          f"{run_config['A1']} - {run_config['A2']} + {run_config['B2']}")
            if self.verbose:
                print(analogy_name)

            res[analogy_name] = self._players_analogy(run_config['A1'], run_config['A2'], run_config['B2'],
                                                      analogy_name=analogy_name)

        if self.export_artifacts:
            if not os.path.exists(PATHS.EXPLAINERS):
                os.makedirs(PATHS.EXPLAINERS)
            for phrase_ in res:
                res[phrase_].columns = [phrase_, 'cosine_sim']
                res[phrase_].to_csv(os.path.join(PATHS.EXPLAINERS, f'Analogy:{phrase_}.csv'))

        return res

    def default_run(self):
        triplets = [
            {'A1': PLAYERS.PIQUE, 'A2': PLAYERS.LUIS_SUAREZ, 'B2': PLAYERS.BENZEMA,
             'analogy_name': 'Defender to striker: Pique - Suarez + Benzema ~ ?'},

            {'A1': PLAYERS.GRIEZMANN, 'A2': PLAYERS.DEMBELE, 'B2': PLAYERS.NEYMAR,
             'analogy_name': 'Dribble skill: Griezmann - Dembele + Neymar ~ ?'},

            {'A1': PLAYERS.JORDI_ALBA, 'A2': PLAYERS.PIQUE, 'B2': PLAYERS.SERGIO_RAMOS,
             'analogy_name': 'Player position: Alba - Pique + Ramos ~ ?'},

            {'A1': PLAYERS.MANE, 'A2': PLAYERS.SALAH, 'B2': PLAYERS.NEYMAR,
             'analogy_name': 'Player position: Mane - Salah + Neymar ~ ?'},

            {'A1': PLAYERS.JORDI_ALBA, 'A2': PLAYERS.NEYMAR, 'B2': PLAYERS.SALAH,
             'analogy_name': 'Player forward position: Alba - Neymar + Salah ~ ?'},

        ]

        return self.players_analogies_analysis(triplets)

class PlayerSkillsExplainer(ModelExplainer):
    '''
    PlayerSkillsExplainer is especially strict with following naming guidelines.
    Alternatively, you can modify the language patterns in language_patterns.py and corresponding namings in:
    explain.py > PlayerSkillsExplainer class.
    '''

    def __init__(self, player_match2vec: Doc2Vec, players_corpus: Corpus, players_metadata: dict,
                 players_vectors: pd.DataFrame,
                 events_data: pd.DataFrame, **kwargs):

        # Data arguments
        self.combined_variations_plot = kwargs.get('combined_variations_plot', False)
        self.entries_std_per_player = {}
        self.plotly_export = kwargs.get('plotly_export', False)
        self._plot = kwargs.get('plot', False)
        self.verbose = kwargs.get('verbose', False)
        self.save_artifacts = kwargs.get('save_artifacts', False)
        self.model_name = kwargs.get('model_name', 'Player2Vec')
        self.players_metadata = players_metadata
        self.players_corpus = players_corpus
        self.player_match2vec = player_match2vec
        self.players_vectors = players_vectors
        self.events_data = events_data

        # Analysis arguments
        self.steps = kwargs.get('steps', 10)
        self.k = kwargs.get('k', 10)
        self.add_baseline_bias = kwargs.get('add_baseline_bias', False)
        self.dimensions_reducer = umap.UMAP().fit(players_vectors)

        # Utils objects
        self.orig_corpus_players = set(list(self.players_vectors.index.copy()))
        self.docs_to_ix = {val: key for key, val in enumerate(self.players_corpus.documents_names)}
        self.player_to_matches = {player_: [doc_ for doc_ in players_corpus.documents_names if player_ in doc_]
                                  for player_ in players_vectors.index}
        self.player_to_matches_docs = {
            player_: [players_corpus.corpus[int(self.docs_to_ix[match_])] for match_ in matches_]
            for player_, matches_ in self.player_to_matches.items()}

        self.positions = events_data[COLUMNS.POSITION].unique()
        self.positions = [val for val in self.positions if isinstance(val, str)]
        self.actions_mapper = {'modify': self.create_player_variation, 'enrich': self.create_player_enriched_variation}

        # Run arguments
        self._player = None
        self._position_name = None
        self._skill_name = None
        self._variation_action = None
        self._skill_improvements = None
        self._sign = None
        self._player_matches = None
        self._baseline_vector = None
        self._last_action = None
        self._next_action = None

        self.baselines_vectors = None
        self.baselines_docs_ix = None

        self.skill_patterns = {}
        self.skill_success_patterns = {}
        self.modify_patterns = {}
        self.enrich_patterns = {}
        self.variations_stats = {}
        self.build_baseline_vectors()
        self.build_patterns()

    def get_all_skills(self):
        return ['dribble', '-dribble', 'shot', '-shot', 'dribble+shot', '-dribble-shot',
                '-right_foot_use', 'pass_to_back', 'high_passes',
                'enrich_dribble', 'enrich_shots', 'enrich_inbox_shots', 'enrich_goals', 'enrich_interceptions',
                'reduce_dribble', 'reduce_shots']

    def build_patterns(self):
        self.skill_patterns['dribble'] = ANDPattern(['<dribble>'], [])
        self.skill_success_patterns['dribble'] = ANDPattern(['=complete'], [])
        self.skill_patterns['-dribble'] = ANDPattern(['<dribble>'], [])
        self.skill_success_patterns['-dribble'] = ANDPattern(['=complete'], [])
        self.skill_patterns['shot'] = ANDPattern(['<shot>'], [])
        self.skill_success_patterns['shot'] = ANDPattern(['=goal'], [])
        self.skill_patterns['-right_foot_use'] = ORPattern(['<pass>', '<shot>'], [])
        self.skill_success_patterns['-right_foot_use'] = ANDPattern(['left_foot'], [])
        self.skill_patterns['pass'] = ANDPattern(['<pass>'], [])
        self.skill_success_patterns['pass'] = ANDPattern([], ['outcome'])
        self.skill_patterns['interception'] = ANDPattern(['<interception>'], [])
        self.skill_success_patterns['interception'] = ORPattern(['success_in_play', 'won'], [])
        self.skill_patterns['enrich_interceptions'] = self.skill_patterns['interception']
        self.skill_success_patterns['enrich_interceptions'] = self.skill_success_patterns['interception']
        self.skill_patterns['dribble+shot'] = self.skill_patterns['shot']
        self.skill_success_patterns['dribble+shot'] = self.skill_patterns['shot']
        self.skill_patterns['-dribble-shot'] = self.skill_patterns['shot']
        self.skill_success_patterns['-dribble-shot'] = self.skill_patterns['shot']
        self.skill_patterns['enrich_dribble'] = self.skill_patterns['dribble']
        self.skill_success_patterns['enrich_dribble'] = self.skill_success_patterns['dribble']
        self.skill_patterns['reduce_dribble'] = self.skill_patterns['dribble']
        self.skill_success_patterns['reduce_dribble'] = self.skill_success_patterns['dribble']
        self.skill_patterns['enrich_shots'] = self.skill_patterns['shot']
        self.skill_success_patterns['enrich_shots'] = self.skill_success_patterns['shot']
        self.skill_patterns['enrich_shots'] = self.skill_patterns['shot']
        self.skill_success_patterns['enrich_shots'] = self.skill_success_patterns['shot']
        self.skill_patterns['reduce_shots'] = self.skill_patterns['shot']
        self.skill_success_patterns['reduce_shots'] = self.skill_success_patterns['shot']
        self.skill_patterns['enrich_inbox_shots'] = self.skill_patterns['shot']
        self.skill_success_patterns['enrich_inbox_shots'] = self.skill_success_patterns['shot']
        self.skill_patterns['enrich_goals'] = self.skill_patterns['shot']
        self.skill_success_patterns['enrich_goals'] = self.skill_success_patterns['shot']
        self.skill_patterns['pass_to_back'] = self.skill_patterns['pass']
        self.skill_success_patterns['pass_to_back'] = ANDPattern(['( v'], [])
        self.skill_patterns['high_passes'] = self.skill_patterns['pass']
        self.skill_success_patterns['high_passes'] = ANDPattern(['high'], [])

        self.modify_patterns['shot'] = [{'pattern': ANDPattern(['<shot>', 'outcome=blocked'], []),
                                         'switch_from': 'outcome=blocked',
                                         'switch_to': 'outcome=goal'},
                                        {'pattern': ANDPattern(['<shot>', 'outcome=wayward'], []),
                                         'switch_from': 'outcome=wayward',
                                         'switch_to': 'outcome=goal'},
                                        {'pattern': ANDPattern(['<shot>', 'outcome=saved'], []),
                                         'switch_from': 'outcome=saved',
                                         'switch_to': 'outcome=goal'},
                                        {'pattern': ANDPattern(['<shot>', 'outcome=off_t'], []),
                                         'switch_from': 'outcome=off_t',
                                         'switch_to': 'outcome=goal'},
                                        ]
        self.modify_patterns['pass_to_back'] = [{'pattern': ANDPattern(['<pass>', '|low'], []),
                                                 'switch_from': '|low',
                                                 'switch_to': '|high'}]

        self.modify_patterns['high_passes'] = [{'pattern': ANDPattern(['<pass>', '|ground'], []),
                                                'switch_from': '|ground',
                                                'switch_to': '|high'},
                                               {'pattern': ANDPattern(['<pass>', '|low'], []),
                                                'switch_from': '|low',
                                                'switch_to': '|high'},

                                               {'pattern': ANDPattern(['<pass>', '|med'], []),
                                                'switch_from': '|med',
                                                'switch_to': '|high'}
                                               ]

        self.modify_patterns['-shot'] = [{'pattern': ANDPattern(['<shot>', 'outcome=goal'], []),
                                          'switch_from': 'outcome=goal',
                                          'switch_to': 'outcome=wayward'}]

        self.modify_patterns['dribble'] = [{'pattern': ANDPattern(['<dribble>', 'outcome=incomplete'], []),
                                            'switch_from': 'incomplete',
                                            'switch_to': 'complete'}]

        self.modify_patterns['-dribble'] = [{'pattern': ANDPattern(['<dribble>', 'outcome=complete'], []),
                                             'switch_from': 'complete',
                                             'switch_to': 'incomplete'}]
        self.modify_patterns['-dribble-shot'] = self.modify_patterns['-dribble'] + self.modify_patterns['-shot']
        self.modify_patterns['dribble+shot'] = self.modify_patterns['dribble'] + self.modify_patterns['shot']

        self.modify_patterns['-right_foot_use'] = [{'pattern': ORPattern(['left_foot'], []),
                                                    'switch_from': 'left_foot',
                                                    'switch_to': 'right_foot'}]
        self.enrich_patterns['enrich_interceptions'] = [{'pattern': ORPattern([f"<pass>"], []),
                                                         '<added_action>': "<interception>",
                                                         'token_builder': self._build_interception,
                                                         'add_before_pattern': True,
                                                         }]
        self.enrich_patterns['reduce_dribble'] = [
            # Give 2-3 cases where we can add dribbles - find manually in the data
            {'<added_action>': "<>", 'pattern': ANDPattern(['<dribble>'], []), 'skip': True},
        ]
        self.enrich_patterns['reduce_shots'] = [
            # Give 2-3 cases where we can add dribbles - find manually in the data
            {'pattern': ANDPattern(['<shot>'], []), 'skip': True, '<added_action>': "<>"}
        ]

        self._skill_name = 'Shot'
        shots_last_action_distribution = self.analyze_action_context()
        self.enrich_patterns['enrich_shots'] = [
            # Give 2-3 cases where we can add dribbles - find manually in the data
            {'pattern': ORPattern([f"<{action_type.lower()}>"], []),
             '<added_action>': "<shot>",
             # token_builder: function that adds the relevant token according to the last success intervention
             'token_builder': self._build_shot,
             'token_builder_args': {},
             } for action_type in shots_last_action_distribution.index[:5]]

        self.enrich_patterns['enrich_inbox_shots'] = [
            # Give 2-3 cases where we can add dribbles - find manually in the data
            {'pattern': ORPattern([f"<{action_type.lower()}>"], []),
             '<added_action>': "<shot>",
             # token_builder: function that adds the relevant token according to the last success intervention
             'token_builder': self._build_shot,
             'token_builder_args': dict(inbox_shot=True),
             } for action_type in shots_last_action_distribution.index[:5]]

        self.enrich_patterns['enrich_goals'] = [
            # Give 2-3 cases where we can add dribbles - find manually in the data
            {'pattern': ORPattern([f"<{action_type.lower()}>"], []),
             '<added_action>': "<shot>",
             'token_builder': self._build_shot,
             'token_builder_args': dict(goal=True),
             } for action_type in shots_last_action_distribution.index[:5]]

        self._skill_name = 'Dribble'
        dribbles_last_action_distribution = self.analyze_action_context()
        self.enrich_patterns['enrich_dribble'] = [{'pattern': ORPattern([f"<{action_type.lower()}>"], []),
                                                   '<added_action>': "<dribble>",
                                                   # token_builder: function that adds the relevant token according to the situation
                                                   # For dribble: Get position of last action, and add a successful dribble
                                                   'token_builder': self._build_dribble,
                                                   'token_builder_args': {},
                                                   } for action_type in dribbles_last_action_distribution.index[:5]]
        for skill in self.get_all_skills():
            self.variations_stats[skill] = {}

    def players_skills_analysis(self, phrases: list):
        '''

        :param phrases: list of dicts: {'action': 'modify'/'enrich',
                                        'skill_name': e.g. 'dribble',
                                        'player_name': str, full & exact name as appeared in the StatsBomb dataset
        :return: no return, just update the Explainer object and produce plots
        '''
        for phrase_ in phrases:
            self._player_matches = None
            self._player = phrase_[COLUMNS.PLAYER_NAME]
            self._position_name = phrase_[COLUMNS.POSITION]
            self._baseline_vector = self.baselines_vectors[self._position_name]

            skills_to_iterate = phrase_['skill_name']
            if not isinstance(skills_to_iterate, list):
                skills_to_iterate = [skills_to_iterate]
            for i, _skill_name in enumerate(skills_to_iterate):
                self._skill_name = _skill_name
                self._sign = '+' if '-' not in _skill_name else '-'

                if 'enrich' in self._skill_name or 'reduce' in self._skill_name:
                    self._variation_action = 'enrich'
                    self._skill_improvements = self.enrich_patterns[self._skill_name]
                else:
                    self._variation_action = 'modify'
                    self._skill_improvements = self.modify_patterns[self._skill_name]

                if 'baseline' in self._player:
                    self._player_matches = self.baselines_docs_ix[self._position_name]

                self.players_vectors = self.players_vectors.drop_duplicates()
                variations_similarities, variations_docs = self._skill_analysis()
                for variation_, variation_docs_ in variations_docs.items():
                    self.variations_stats[self._skill_name][variation_] = self.calc_skill_stats( \
                        variation_docs_, self.skill_patterns[self._skill_name],  # event pattern
                        self.skill_success_patterns[self._skill_name])  # event success pattern

            # Calculate std of vector representation player's variations. Display vec entries sorted by std DESC
            for skill in skills_to_iterate:
                if len(self.variations_stats[skill]) == 0:
                    continue
                rel_players_vectors = self.players_vectors.copy()
                rel_players_vectors['groupby'] = [val.split(self.players_corpus.separator)[0] for val in
                                                  rel_players_vectors.index.copy()]
                rel_players_vectors = rel_players_vectors.loc[
                    [ix for ix in self.players_vectors.index.copy() if skill in ix]]
                self.entries_std_per_player[skill] = rel_players_vectors.std(axis=0).sort_values(ascending=False)
                if self.verbose:
                    print(f"For the {skill} skill, most changes representation indexes:")
                    print(self.entries_std_per_player[skill])
        if self.combined_variations_plot:
            variations_names = [val for val in self.players_vectors.index if '_enrich_' in val or '_modify_' in val]
            plot_embeddings(self.players_vectors, variations_names,
                            docs_features=True,
                            docs_data=self.players_metadata,
                            doc_name_separator=self.players_corpus.separator, model_name=self.model_name,
                            highlight_selected=False,
                            title=f'UMAP projection of the skill players and players variations',
                            dimensions_reducer=self.dimensions_reducer,
                            show=self.verbose, save_fig=self.save_artifacts, plotly_export=self.plotly_export)

        fig = make_subplots(rows=2, cols=1, shared_xaxes=True, shared_yaxes=False,
                            subplot_titles=['Actions modifications, Actions enrichments'],
                            vertical_spacing=0.1, horizontal_spacing=0.01)

        for i, skill in enumerate(self.entries_std_per_player):
            if len(self.entries_std_per_player[skill]) > 0:
                df = self.entries_std_per_player[skill].reset_index()
                df.columns = ['dimension', 'std']
                if 'enrich' in skill or 'reduce' in skill:
                    curr_row = 2
                    action = 'enriched'
                else:
                    curr_row = 1
                    action = 'modified'

                fig.add_trace(go.Bar(x=df['dimension'], y=df['std'], name=skill), row=curr_row, col=1)

                # Set x-axis title
                fig.update_xaxes(title_text="Representation dimensions", row=curr_row, col=1)
                # Set y-axes titles
                fig.update_yaxes(title_text="<b>STD</b>", row=curr_row, col=1)

        fig.update_layout(title_text=f'Dimensions STD VS. skill changed')
        fig.show()

    def analyze_action_context(self, next_action_distribution=False):
        # Understand when an action happens
        actions = self.events_data.loc[self.events_data[COLUMNS.ACTION_TYPE] == self._skill_name, 'index'].copy()
        last_actions = []
        next_actions = []
        for action_ in actions:
            next_action = self.events_data.at[action_ + 1, COLUMNS.ACTION_TYPE]
            next_actions.append(next_action)
            last_action = self.events_data.at[action_ - 1, COLUMNS.ACTION_TYPE]
            last_actions.append(last_action)

        if next_action_distribution:
            next_actions_distribution = pd.Series(next_actions).value_counts()
            next_actions_distribution = next_actions_distribution / next_actions_distribution.sum()
            if self.verbose:
                print(f'\nAction distribution after {self._skill_name}s (ONE ACTION):')
                print(next_actions_distribution)
                print(f'\nAction distribution after {self._skill_name}s (ONE ACTION, NORMED):')
                print(next_actions_distribution)
            return next_action_distribution

        last_action_distribution = pd.Series(last_actions).value_counts()
        last_action_distribution = last_action_distribution / last_action_distribution.sum()
        if self.verbose:
            print(f'\nAction distribution before {self._skill_name}s (ONE ACTION):')
            print(last_action_distribution)

        return last_action_distribution

    def infer_in_vocab_player(self, player_matches):
        player_new_vectors = []
        for i, match_ in enumerate(player_matches):
            if isinstance(match_, list):
                match_doc = match_[:]
            elif isinstance(match_, int):
                match_doc = self.players_corpus.corpus[match_]
            else:
                raise NotImplementedError(f"Illegal type value for match_doc - {type(match_)}")
            # Modify all matches of this player
            inferred_vector = estimate_doc_vector(self.player_match2vec, match_doc, steps=self.steps,
                                                  repetitions=self.k, norm=False)
            player_new_vectors.append(inferred_vector)

        player_vec = np.vstack(player_new_vectors).mean(axis=0)
        return player_vec

    def _get_players_in_position(self):
        # Find all players matching this position
        players_in_position = self.events_data.loc[self.events_data[COLUMNS.POSITION] == self._position_name,
                                                   [COLUMNS.PLAYER_NAME, COLUMNS.MATCH_ID]].drop_duplicates()
        players_in_position = players_in_position.apply(
            lambda x: x[COLUMNS.PLAYER_NAME] + self.players_corpus.separator + str(x[COLUMNS.MATCH_ID])
                      + '.json' if '.json' not in str(x[COLUMNS.MATCH_ID]) else '', axis=1)
        players_in_position = players_in_position.to_list()

        # Calc avg num docs for players in position.
        matches_per_player = self.events_data[[COLUMNS.PLAYER_NAME, COLUMNS.MATCH_ID]].groupby(
            by=[COLUMNS.PLAYER_NAME]).count()
        num_docs_to_use = matches_per_player[COLUMNS.MATCH_ID].median()
        return players_in_position, num_docs_to_use

    def calc_skill_stats(self, _player_docs: [list], event_pattern, success_pattern) -> (np.array, pd.Series):
        '''
        Calculate frequency ang average success rate of the skill
        :param event_pattern: regex pattern to match for counting the event as relevant
        :param success_pattern: regex pattern to identify if the event was successful
        '''
        mean_action_counts = []
        mean_success_rate = []
        action_count = 0
        success_count = 0
        for doc in _player_docs:
            for token_ix in doc:
                token_ = self.players_corpus.ix_2_token[token_ix]
                if _search(event_pattern, token_):
                    action_count += 1
                    if _search(success_pattern, token_):
                        success_count += 1
            mean_action_counts.append(action_count)
            mean_success_rate.append(0 if action_count == 0 else success_count / action_count)
        return np.mean(mean_action_counts), np.mean(mean_success_rate)

    def create_player_variation(self, player_name: str, interventions: list, limit_interventions=None,
                                player_matches=None, inference_bias_vector=None) -> (np.array, pd.Series):
        '''
        Create a new player after executing a list of interventions in his matches.
        :param player_name: name of the player to improve, str
        :param interventions: list of interventions to apply on the player matches
        :param limit_interventions: if None - no limitation, it prevents from making more than limit_interventions changes
        :param inference_bias_vector: vector of inference bias to add if add_baseline_bias is true
        - Globalparam normalize_vector: if True, inferred vector is normalized after inference [and adding bias]
        :return: numpy array of the new player variation, and the k most similar players to it.
        '''
        # Get all player's matches
        if player_matches is None:
            player_matches = self.player_to_matches_docs[player_name]
        _variation_docs = []
        player_new_vectors = []
        interventions_counts = []
        oov_counts = []
        for i, match_ in enumerate(player_matches):
            if isinstance(match_, str):
                match_doc = self.players_corpus.corpus[int(self.docs_to_ix[match_])]
            elif isinstance(match_, list):
                match_doc = match_[:]
            elif isinstance(match_, int):
                match_doc = self.players_corpus.corpus[match_]
            else:
                raise NotImplementedError(f"Illegal type value for match_doc - {type(match_)}")
            # Modify all matches of this player
            player_variation_doc, interventions_count, oov_count = modify_doc(match_doc, interventions,
                                                                              self.players_corpus,
                                                                              limit_interventions=limit_interventions)
            interventions_counts.append(interventions_count)
            oov_counts.append(oov_count)
            _variation_docs.append(player_variation_doc[:])

            # Learn the variation's representation
            player_new_vector = estimate_doc_vector(self.player_match2vec, player_variation_doc, steps=self.steps,
                                                    repetitions=self.k,
                                                    norm=False)

            player_new_vectors.append(player_new_vector)

        player_variation_vec = np.vstack(player_new_vectors).mean(axis=0)

        if self.add_baseline_bias and inference_bias_vector is not None:
            player_variation_vec = player_variation_vec + inference_bias_vector

        distances = pd.DataFrame(self.players_vectors.index.copy(), columns=['player_name'])
        distances['phrase_similarity'] = distances['player_name'].apply( \
            lambda RHS: np.dot(player_variation_vec, self.players_vectors.loc[RHS]) / (
                    np.linalg.norm(player_variation_vec) * np.linalg.norm(self.players_vectors.loc[RHS])))

        if self.verbose:
            print(
                f"{player_name} variation completed with {len(player_matches)} matches "
                f"and {np.mean(interventions_counts)} interventions on avg"
                f"(OOV avg: {np.mean(np.mean(oov_counts))})")
        return player_variation_vec, distances.sort_values(by=['phrase_similarity'], ascending=False), _variation_docs

    def create_player_enriched_variation(self, player_name: str, _interventions: list, player_matches=None,
                                         inference_bias_vector=None):
        '''
        Creates a player variation by enriching his documents - applying
        :param player_name: str
        :param _interventions: list of regex patterns to match. if match - apply _enricher function on it
        :type _interventions: list of dicts
                                - 'token_builder': function that adds the relevant token according to the last success intervention
        :param player_matches: list of lists. Inner lists are the player documents (e.g, for baseline players that
                                their games are not in the player_to_matches object.
                                Each inner list is a list of tokens indexes (not strs).
                                :param inference_bias_vector: vector of inference bias to add if add_baseline_bias is true
        - Globalparam normalize_vector: if True, inferred vector is normalized after inference [and adding bias]
        :return: new eneriched doc (list), most similar docs (df) and player_new_docs
        '''
        # Get all player's matches
        if player_matches is None:
            player_matches = self.player_to_matches_docs[player_name]

        player_new_vectors = []
        player_new_docs = []
        num_interventions = []
        oov_counts = []
        for i, match_ in enumerate(player_matches):
            if isinstance(match_, str):
                match_doc = self.players_corpus.corpus[int(self.docs_to_ix[match_])]
            elif isinstance(match_, list):
                match_doc = match_[:]
            elif isinstance(match_, int):
                match_doc = self.players_corpus.corpus[match_]
            else:
                raise NotImplementedError(f"Illegal type value for match_doc - {type(match_)}")

            player_new_docs.append(match_doc[:])
            # Modify all matches of this player
            new_match_doc, n_interventions, oov_count = enrich_doc(match_doc, _interventions, self.players_corpus,
                                                                   limit_interventions=None)
            oov_counts.append(oov_count)
            num_interventions.append(n_interventions)
            new_match_enriched_doc = estimate_doc_vector(self.player_match2vec, new_match_doc, steps=self.steps,
                                                         repetitions=self.k,
                                                         norm=False)
            player_new_vectors.append(new_match_enriched_doc)

        player_new_vec = np.vstack(player_new_vectors).mean(axis=0)

        if self.add_baseline_bias and inference_bias_vector is not None:
            player_new_vec = player_new_vec + inference_bias_vector

        distances = pd.DataFrame(self.players_vectors.index.copy(), columns=['player_name'])
        distances['phrase_similarity'] = distances['player_name'].apply( \
            lambda RHS: np.dot(player_new_vec, self.players_vectors.loc[RHS]) / (
                    np.linalg.norm(player_new_vec) * np.linalg.norm(self.players_vectors.loc[RHS])))

        if self.verbose:
            print(
                f"{player_name} variation completed with {len(player_matches)} matches "
                f"and {np.mean(np.mean(num_interventions))} interventions on avg "
                f"(OOV avg: {np.mean(np.mean(oov_counts))})")
        return player_new_vec, distances.sort_values(by=['phrase_similarity'],
                                                     ascending=False).head(self.k), player_new_docs

    def get_player_inference_bias_vector(self, _player, player_inferred_vector, players_vectors):
        if _player in players_vectors:
            inference_bias_vector = player_inferred_vector.copy() - players_vectors.loc[_player]
        else:
            inference_bias_vector = np.zeros_like(player_inferred_vector)
        return inference_bias_vector

    def build_baseline_vectors(self):
        # Create a baseline player for each position, so we can give attributes to the player by comparing them to the
        #   appropriate baseline player
        self.baselines_vectors = {}
        self.baselines_docs_ix = {}
        for position_ in self.positions:
            self._position_name = position_
            players_in_curr_position, avg_matches_count = self._get_players_in_position()
            self.baselines_vectors[position_], __, self.baselines_docs_ix[position_] = create_baseline_player(
                self.player_match2vec,
                players_in_curr_position,
                avg_matches_count,
                self.players_corpus)

    def _build_dribble(self, _last_action):
        location = _last_action.split('<')[0]
        x = location[1]

        if int(x) < 3:
            # Fix location to midfield/ attacking position - randomly chose between 4 and 5
            x = str(int(np.random.choice([4, 5])))
            location = f"({x}{location[2:]}"
        return location + "<dribble>:|outcome=complete"

    def _build_shot(self, _last_action, goal=False, foot='right', inbox_shot=False):
        outcome = np.random.choice(['off_t', 'saved', 'goal'], p=[0.45, 0.45, 0.1]) if not goal else 'goal'
        technique = 'normal'
        shot_type = 'open_play'
        body_part = f'{foot}_foot'
        if '<pass>' not in _last_action:
            # Create shot from same location
            location = _last_action.split('<')[0]
        else:
            # Cannot pass to himself, randomize location
            x = str(int(np.random.choice([4, 5]))) if not inbox_shot else '5'
            y = np.random.choice([2, 3, 4]) if not inbox_shot else '3'
            location = f"({x}/5,{y}/5)"

        return location + f"<shot>:|{technique}|outcome={outcome}|body_pa={body_part}|type_na={shot_type}"

    def _build_interception(self, next_action):
        outcome = np.random.choice(['success_in_play', 'won'])
        location = next_action.split('<')[0]
        return location + f"<interception>:|outcome={outcome}"

    def _skill_analysis(self):
        '''
        Perform skill analysis on arguments saved in object self
        '''
        # Create few improved versions of _player with different p and show it on the Player2Vec plot
        players_to_highlight = ['baseline', self._player]
        _variations_similarities = []
        _variations_docs = {}

        # Add the same player, but send to inference to detect bias on inference
        _player_inferred_vec, most_similar, _player_inferred_docs = self.create_player_variation( \
            self._player, [], player_matches=self._player_matches)

        inference_bias_vector = self.get_player_inference_bias_vector(self._player, _player_inferred_vec,
                                                                      self.players_vectors)
        variation_name = f'_{self._player}_inference'
        _columns = [variation_name, f"{variation_name}:similarity"]
        _variations_docs[variation_name] = _player_inferred_docs
        _variations_similarities.append(most_similar)
        players_to_highlight.append(variation_name)
        if variation_name not in self.players_vectors.index:
            self.players_vectors = self.players_vectors.append(pd.Series(_player_inferred_vec, name=variation_name))

        if self._variation_action == 'modify':
            p_values = np.arange(0.1, 1.1, 0.1)
        else:
            p_values = np.concatenate((np.arange(0.02, 0.1, 0.02), np.arange(0.1, 0.6, 0.1)))

        for p in p_values:
            # Transform each action with prob p
            for _skill_improve in self._skill_improvements:
                _skill_improve['probability'] = p
            if self.verbose:
                print(f'Creating {self._player} {self._variation_action} variation for skill'
                      f' {self._skill_name} with p={p}')
            players_vectors = self.players_vectors.drop_duplicates()
            variation_vec, most_similar, variation_docs = self.actions_mapper[self._variation_action]( \
                self._player, self._skill_improvements,
                inference_bias_vector=inference_bias_vector,
                player_matches=self._player_matches)

            variation_name = f"{SHORTNAMES.get(self._player, self._player)}_{self._variation_action}" \
                             f"_{self._sign}{self._skill_name}_{np.round(p, 2)}"
            _variations_docs[variation_name] = variation_docs[:]
            # Copy player metadata to his variation
            if self._player in self.players_metadata:
                self.players_metadata[variation_name] = self.players_metadata[self._player].copy()
            else:
                # Variation of a baseline player / artificial player / player without metadata -> create generic
                self.players_metadata[variation_name] = {'Unnamed: 0': 0, 'player_id': 0000,
                                                         'player_nickname': variation_name, 'jersey_number': 0,
                                                         'country_id': 0, 'country_name': 'Artificial',
                                                         'player_name_lower': variation_name,
                                                         'position_name': self._position_name,
                                                         'team_name': 'Artificial'}

            # Analyze most similar players
            _variations_similarities.append(most_similar)
            if self.verbose:
                print(
                    f'Most similar players to {self._player} skill = {self._variation_action} {self._skill_name} '
                    f'and p={p} - WITH VARIATIONS:')
                print(most_similar.head(self.k))
                print(
                    f'Most similar players to {self._player} skill = {self._variation_action} {self._skill_name} '
                    f'and p={p}: - WITHOUT VARIATIONS')
                print(most_similar[most_similar[COLUMNS.PLAYER_NAME].isin(self.orig_corpus_players)].head(self.k))
            # Add the player variation to future plots
            players_to_highlight.append(variation_name)
            if variation_name not in players_vectors.index:
                self.players_vectors = players_vectors.append(pd.Series(variation_vec, name=variation_name))

        # Show similarities to each version so we will see the original player ger further and others closer
        if self.verbose:
            print(f'\nSkill analysis for {self._variation_action} {self._player} at skill {self._skill_name}')
        _variations_similarities = pd.concat(_variations_similarities, axis=1)
        for _p in p_values:
            _columns.extend([f'{self._variation_action} p={np.round(_p, 2)}:{COLUMNS.PLAYER_NAME}',
                             f'{self._variation_action} p={np.round(_p, 2)}:similarity'])
        _variations_similarities.columns = _columns
        _variations_similarities.to_csv(os.path.join(PATHS.EXPLAINERS,
                                                     f"most_similar_{self._player}_{self._variation_action}_"
                                                     f"skill_{self._skill_name}.csv"))

        # Plot embeddings for all new vectors + baseline player
        if f'baseline_{self._position_name}' not in self.players_vectors.index:
            self.players_vectors = self.players_vectors.append(
                pd.Series(self._baseline_vector, name=f'baseline_{self._position_name}'))

        if self._plot:
            self.players_vectors.index = [SHORTNAMES.get(ix, ix) for ix in self.players_vectors.index]
            plot_embeddings(self.players_vectors, players_to_highlight, docs_features=True,
                            docs_data=self.players_metadata,
                            doc_name_separator=self.players_corpus.separator, model_name=self.model_name,
                            highlight_selected=False,
                            title=f'UMAP projection of the skill {self._skill_name} for player {self._player}',
                            dimensions_reducer=self.dimensions_reducer,
                            show=self._plot, save_fig=self.save_artifacts, plotly_export=self.plotly_export)
        return _variations_similarities, _variations_docs

class LinearDocExplainer(ModelExplainer):
    def __init__(self, player_match2vec: Doc2Vec, players_corpus: Corpus, players_vectors: pd.DataFrame,
                 events_data: pd.DataFrame, **kwargs):
        '''
        :param player_match2vec: Doc2Vec model of player-to-match
        :param players_corpus: Corpus object of the player_match2vec model
        :param players_vectors: pd.DataFrame where the index is players name and values are their Player2Vec representation
        :param events_data: pd.DataFrame of events data, in StatsBomb data format
        :param kwargs:
            - repetitions: number of times to infer docs and average the results, for mitigating randomness
        '''
        # Data arguments
        self.min_matches_per_player = kwargs.get('', 10)
        self.verbose = kwargs.get('verbose', True)
        self.players_corpus = players_corpus
        self.player_match2vec = player_match2vec
        self.players_vectors = players_vectors
        self.events_data = events_data

        # Analysis arguments
        self.steps = kwargs.get('steps', 10)
        self.k = kwargs.get('k', 10)
        self.repetitions = kwargs.get('repetitions', 10)

        # supported_skills Dict:
        #   key=skill: str, value=function that return True/False if given token is relevant to skill
        self._supported_skills = {'scoring': is_normal_goal_token,
                                  'in_box_scoring': lambda token: re.match(in_box_scoring_pattern, token),
                                  'out_box_scoring': lambda token: re.match(out_box_scoring_pattern, token),
                                  'dribbling': lambda token: re.match(dribble_pattern, token),
                                  'flank_dribbling': lambda token: re.match(flank_dribble_pattern, token),
                                  'pressure': lambda token: re.match(forward_pressure_pattern, token),
                                  'carry': lambda token: '<carry>' in token,
                                  'offsides': lambda token: '<offside>' in token
                                  }
        self.skills_docs = {}
        self.skills_vectors = {}
        self.build_skills_documents()

    def get_supported_skills(self):
        return list(self._supported_skills.keys())

    def build_skills_documents(self):
        for skill, func in self._supported_skills.items():
            # Building artificial docs #
            self.skills_docs[skill] = self.build_artificial_doc(func)
            self.skills_vectors[skill] = estimate_doc_vector(self.player_match2vec, self.skills_docs[skill],
                                                             steps=self.steps, repetitions=self.repetitions)

    def build_artificial_doc(self, condition) -> list:
        '''
        Creates artificial Doc2Vec document to serve as an explainer
        :param condition: lambda function to apply on each word in the vocabulary
        :type condition: function
        '''
        relevant_tokens = [token for token in self.players_corpus.vocabulary if condition(token)]
        return relevant_tokens

    def default_run(self):
        run_arguments = [{COLUMNS.PLAYER_NAME: PLAYERS.INIESTA, 'skills': ['scoring', 'in_box_scoring',
                                                                           'outbox_scoring', '-dribbling',
                                                                           ['outbox_scoring', '-dribbling']]},
                         {COLUMNS.PLAYER_NAME: PLAYERS.NEYMAR, 'skills': ['-dribbling',
                                                                          '-flank_dribbling',
                                                                          ['-carry', '-dribbling']]},
                         {COLUMNS.PLAYER_NAME: PLAYERS.BUSQUETS,
                          'skills': ['dribbling', 'scoring', '-pressure', 'in_box_scoring', 'out_box_scoring']},

                         {COLUMNS.PLAYER_NAME: PLAYERS.LUIS_SUAREZ,
                          'skills': ['-scoring', '-inbox_scoring', '-pressure']},

                         {COLUMNS.PLAYER_NAME: PLAYERS.GRIEZMANN,
                          'skills': ['dribbling', 'flank_dribbling', '-pressure']}
                         ]
        return self.players_actions_analogies(run_arguments)

    def players_actions_analogies(self, run_arguments: [dict]) -> (dict, dict):
        '''

        :param run_arguments: {COLUMNS.PLAYER_NAME: PLAYERS.GRIEZMANN, 'skills': [list of strings or nested lists]
        :type run_arguments:
        :return: (1, 2):
            1. distances dict: {f"{player}_{skill}": pd.DataFrame of k most similar players, ...}
            2. dict: {f"{player}_{str(skill)}": linear combination of player and skills vectors [np.array]
        :rtype: (dict, dict)
        '''
        players_vectors = {}
        players_w_skills = {}

        def _iterate_skill(skill):
            neg_factor = 1
            if skill.startswith('-'):
                neg_factor = -1
                skill = skill[1:]
            return self.skills_vectors[skill] * neg_factor

        for _run in run_arguments:
            player = _run[COLUMNS.PLAYER_NAME]
            # Get player vector
            player_vector = np.array(self.players_vectors.loc[player])
            players_vectors[player] = {}
            player_vector_norm = player_vector / np.linalg.norm(player_vector)
            players_vectors[player]['player_vector'] = player_vector
            players_vectors[player]['player_vector_norm'] = player_vector_norm

            for _skill in _run['skills']:
                players_w_skills[f"{player}_{_skill}"] = players_vectors[player]['player_vector_norm']
                if isinstance(_skill, list):
                    for skill_i in _skill:
                        players_w_skills[f"{player}_{str(_skill)}"] += _iterate_skill(skill_i)
                else:
                    players_w_skills[f"{player}_{_skill}"] += _iterate_skill(_skill)

        # Count number of matches per player in order to keep player with at least min_matches_per_player
        matches_per_player = self.events_data[['player_name', 'match_id']].drop_duplicates().groupby(
            by=['player_name']).count() \
            .to_dict(orient='index')

        distances = {}
        for item_name, item in players_w_skills.items():
            distances[item_name] = {}
            for i, (player_i, vec_i) in enumerate(self.players_vectors.iterrows()):
                if matches_per_player[player_i][COLUMNS.MATCH_ID] >= self.min_matches_per_player:
                    distances[item_name][player_i] = np.dot(vec_i, item) / (
                            np.linalg.norm(vec_i) * np.linalg.norm(item))

            similarities = pd.DataFrame.from_dict(distances[item_name], orient='index')
            similarities.columns = ['cosine similarity']
            if self.verbose:
                print(f'Phrase = {item_name}. K most similar by cosine:')
                print(similarities.sort_values(by=['cosine similarity'], ascending=False).head(self.k))
                print()

        return distances, players_w_skills

def Player2Vec_std_analysis(players_metadata: dict, players_to_highlight: list = None, plotly_export=False,
                            save_artifacts=False, _plot=False):
    '''
    STD for Player2Vec figure to capture the span in space
    :param players_metadata: dict, metadata of all players (documents)
    :param players_to_highlight: list of players to highlight in plots, adding label and coloring
    :param  plotly_export: Whether to export plotly figures to plotly studio (see https://chart-studio.plotly.com)
    :param save_artifacts: whether to save the figure to artifacts directory or not
    '''
    model_name = "Player2Vec"
    corpus_obj_path = os.path.join(MODELS_ARTIFACTS, f'{model_name}_corpus.pickle')
    with open(corpus_obj_path, 'rb') as f:
        players_corpus = pickle.load(f)

    player_2_vec_model = Doc2Vec.load(os.path.join(MODELS_ARTIFACTS, f'{model_name}.model'))

    embedded_vocab = {}
    for i, doc_ in enumerate(players_corpus.documents_names):
        embedded_vocab[doc_] = player_2_vec_model.infer_vector(players_corpus.corpus[i])

    embedded_vocab = pd.DataFrame.from_dict(embedded_vocab, orient='index')
    embedded_vocab = embedded_vocab.reset_index()
    embedded_vocab[COLUMNS.PLAYER_NAME] = embedded_vocab['index'].apply(
        lambda ix: ix.split(players_corpus.separator)[0])
    players_embeddings = embedded_vocab.groupby(COLUMNS.PLAYER_NAME).std()

    # Remove NAs (not enough matches for calculation)
    players_embeddings = players_embeddings.dropna()

    model_name += ' Variance'
    plot_embeddings(players_embeddings, players_to_highlight, docs_features=True, docs_data=players_metadata,
                    doc_name_separator=players_corpus.separator, model_name=model_name,
                    color_attr=COLUMNS.POSITION, plotly_export=plotly_export, show=_plot,
                    title=f'UMAP projection of the {model_name}', save_fig=save_artifacts)

def analyze_vector_dimensions_semantics(embeddings_df: pd.DataFrame, k=10) -> pd.DataFrame:
    '''
    Return top k docs/tokens with highest value of each index of the representations
    :param embeddings_df: index = token/doc, columns = vector representation dimensions
    :param k: number of top (samples with highest values) and bottom (samples with lowest values) results to return
    :return: top matching tokens/docs concatenated with the worst matching tokens/doct
    '''
    top_results = {}
    bottom_results = {}

    for col in embeddings_df.columns:
        curr_df = embeddings_df.sort_values(by=col, ascending=False)
        top_results[col] = list(curr_df.head(k).index)[:]
        bottom_results[col] = list(curr_df.tail(k).index)[:]

    top_results = pd.DataFrame.from_dict(top_results, orient='index')
    top_results.columns = [f"Top_{i}" for i in range(top_results.shape[1])]
    bottom_results = pd.DataFrame.from_dict(bottom_results, orient='index')
    bottom_results.columns = [f"Bottom_{-i}" for i in range(top_results.shape[1], 0, -1)]
    return pd.merge(top_results, bottom_results, left_index=True, right_index=True)


text:
# -- --------------------------------------------------------------------------------------------------- -- #
# -- Proyecto: Proyecto final de materia Análisis Estadístico Multivariado                               -- #
# -- Codigo: funciones.py - codigo con funciones de modelos y operaciones complejas                      -- #
# -- Repositorio: https://github.com/IFFranciscoME/FinTechLab/tree/master/MCD-ITESO/AEM/aem_proyecto     -- #
# -- Autor: Francisco ME                                                                                -- #
# -- --------------------------------------------------------------------------------------------------- -- #

import numpy as np                                     # funciones numericas
import pandas as pd                                    # dataframes y utilidades
from statsmodels.tsa.api import acf, pacf              # funciones de econometria
from statsmodels.formula.api import ols                # herramientas estadisticas: modelo lineal con ols
from sklearn.preprocessing import StandardScaler       # estandarizacion de variables
from sklearn.decomposition import PCA                  # analisis de componentes principales (PCA)
import statsmodels.api as sm                           # utilidades para modelo regresion lineal
from sklearn.model_selection import train_test_split   # separacion de conjunto de entrenamiento y prueba
from sklearn.metrics import mean_squared_error         # error cuadratico medio

import mass_ts as mts                                  # Clustering para series de tiempo
import warnings
warnings.filterwarnings("ignore", category=UserWarning, module='mass_ts')

pd.set_option('display.max_rows', None)                # sin limite de renglones maximos para mostrar pandas
pd.set_option('display.max_columns', None)             # sin limite de columnas maximas para mostrar pandas
pd.set_option('display.width', None)                   # sin limite el ancho del display
pd.set_option('display.expand_frame_repr', False)      # visualizar todas las columnas de un dataframe
pd.options.mode.chained_assignment = None              # para evitar el warning enfadoso de indexacion

# -- ------------------------------------------------------------------------------ FUNCION: STSC - MASS -- #
# -- ------------------------------------------------------------------------------------ Version manual -- #

def f_stsc_mass(p_precios, p_calendario, p_ventana):
    """
    :param: p_precios : dataframe : precios OHLC para crear una serie
    :param: p_calendario : dataframe :
    :param: p_indicadores : dataframe :
    :param: p_ventana : int :

    :return:

    p_precios = df_pe_m1
    p_indicadores = res3_anova
    p_calendario = df_ce_w
    p_ventana = 30

    """

    # de la serie de precios completa de 1 minuto, tomando una sub-serie de ventana T de la reacción
    # del precio ante el comunicado de un indicador que paso la prueba de autoanova, quiero saber si
    # hubo otras veces donde se repitió el mismo patrón y si las hubo, que sean de inmediato posterior
    # a las fechas cercanas al comunicado del indicador en otras fechas.

    fechas = list()
    sub_datos = p_calendario

    # buscar motifs en todas las fechas en las que se publico el indicador y dio el escenario
    # aprobado por anova

    for i in range(0, len(sub_datos['timestamp'])):
        # print('buscando patrones para: ' + indicador + ' en ' + str(sub_datos['timestamp'][i]))

        # fecha inicial de serie query es un evento arbitrario del calendario
        fecha_ini = sub_datos['timestamp'][i]
        # se toma el timestamp de precios igual a timestamp del primer escenario del indicador
        ind_ini = np.where(p_precios['timestamp'] == fecha_ini)[0]
        # fecha final es la fecha inicial mas un tamaño de ventana arbitrario
        ind_fin = ind_ini + p_ventana
        # se construye la serie query
        serie_q = p_precios.iloc[ind_ini[0]:ind_fin[0], :]
        # serie_q = np.array((serie_q['close'] - serie_q['open']) * 10000)
        serie_q = np.array(serie_q['close'])
        serie_q = serie_q/max(serie_q)

        # se construye la serie de busqueda (un array de numpy de 1 dimension)
        serie = np.array(p_precios['close'])[7:]
        serie = serie/max(serie)

        # parametros del algoritmo
        # tamaño de ventana para iterar la busqueda = tamaño de query
        batch_size = p_ventana*100
        # regresar los Top X casos que "mas se parezcan" = Cantidad total de publicaciones de indicador
        top_matches = 100
        # regresar los indices y las distancias
        mass_indices, mass_dists = mts.mass2_batch(serie, serie_q, batch_size=batch_size,
                                                   top_matches=top_matches, n_jobs=3)

        # obtener las fechas de los indices regresados fecha inicial del motif detectado y
        # fecha final segun amplitud p_ventana.
        mass_fechas = [p_precios['timestamp'][mass_indices[i]] for i in range(0, len(mass_indices))]

        # cada cada fecha propuesta por el MASS como fecha inicial de un motif con similitud, revisar si
        # esta fecha propuesta coincide con otra fecha de publicación de indicador con el mismo escenario
        fecha_encontrada = list()
        for k in range(0, len(mass_fechas)):
            encuentros = [i for i, j in enumerate(list(sub_datos['timestamp'])) if j == mass_fechas[k]]
            if len(encuentros) != 0 and mass_fechas[k] != fecha_ini:
                fecha_encontrada.append(mass_fechas[k])  # mismo patron entre precios y serie_q con mass

        fechas.append(fecha_encontrada)

    return fechas

# -- ------------------------------------------------- FUNCION: Clasificacion de escenarios de indicador -- #
# -- ------------------------------------------------------------------------------------ Version manual -- #

def f_escenario(p0_datos):
    """
    :param: pd.DataFrame : DataFrame : calendario economico con columnas: timestamp, name,
                                                                     actual, consensus, previous
    :return:
    """

    datos = p0_datos

    # criterio para rellenar datos faltantes 'nan'
    # cuando falta en consensus
    nan_consensus = datos.index[np.isnan(datos['consensus'])].tolist()

    # asignarle a consensus lo que tiene previous
    datos['consensus'][nan_consensus] = datos['previous'][nan_consensus]

    # inicializar la columna escenario, habra los siguientes: A, B, C, D
    datos['escenario'] = 'X'

    # -- -- A: actual >= previous & actual >= consensus & consensus >= previous
    datos['escenario'][((datos['actual'] >= datos['previous']) &
                        (datos['actual'] >= datos['consensus']))] = 'A'

    # -- -- B: actual >= previous & actual >= consensus & consensus < Precious
    datos['escenario'][((datos['actual'] >= datos['previous']) &
                        (datos['actual'] < datos['consensus']))] = 'B'

    # -- -- C: actual >= previous & actual < consensus & consensus >= previous
    datos['escenario'][((datos['actual'] < datos['previous']) &
                        (datos['actual'] >= datos['consensus']))] = 'C'

    # -- -- D: actual >= previous & actual < consensus & consensus < previous
    datos['escenario'][((datos['actual'] < datos['previous']) &
                        (datos['actual'] < datos['consensus']))] = 'D'

    return datos

# -- ---------------------------------------- FUNCION: Generacion de variables EXOGENAS series de tiempo -- #
# -- ------------------------------------------------------------------------------------ Version manual -- #

def f_anova(p_datos_ce, p_datos_ph):
    """
    :param p_datos_ph:
    :param p_datos_ce: DataFrame : calendario economico con columnas: timestamp, name,
                                                                     actual, consensus, previous
    :return:

    p_datos_ce = df_ce_w
    p_datos_ph = df_pe_m1
    """

    def f_reaccion(p0_i, p1_ad, p2_ph, p3_ce):
        """
        :param p0_i: int : indexador para iterar en los datos de entrada
        :param p1_ad: int : cantidad de precios futuros que se considera la ventana (t + p1_ad)
        :param p2_ph: DataFrame : precios en OHLC con columnas: timestamp, open, high, low, close
        :param p3_ce: DataFrame : calendario economico con columnas: timestamp, name,
                                                                     actual, consensus, previous

        :return: resultado: diccionario con resultado final con 3 elementos como resultado

        # debugging
        p0_i = 16
        p0_ad = 30
        p2_ph = pd.DataFrame({'timestamp': 2009-01-06 05:00:00,
                              'open': 1.3556, 'high': 1.3586, 'low': 1.3516, 'close': 1.3543})
        p3_ce = pd.DataFrame({})
        """
        # print(p0_i)
        # print(' fecha ce: ' + str(p3_ce['timestamp'][p0_i]))

        # Encontrar indice donde el timestamp de precios sea igual al del calendario
        indice_1 = np.where(p2_ph['timestamp'] == p3_ce['timestamp'][p0_i])[0][0]
        indice_2 = indice_1 + p1_ad
        ho = round((max(p2_ph['high'][indice_1:indice_2]) - p2_ph['open'][indice_1]) * 10000, 2)
        ol = round((p2_ph['open'][indice_1] - min(p2_ph['low'][indice_1:indice_2])) * 10000, 2)
        co = round((p2_ph['close'][indice_1] - min(p2_ph['open'][indice_1:indice_2])) * 10000, 2)

        # diccionario con resultado final
        resultado = {'ho': ho, 'ol': ol, 'co': co}

        return resultado

    # cantidad de precios a futuro a considerar
    psiguiente = 30

    # reaccion del precio para cada escenario
    d_reaccion = [f_reaccion(p0_i=i, p1_ad=psiguiente, p2_ph=p_datos_ph, p3_ce=p_datos_ce)
                  for i in range(0, len(p_datos_ce['timestamp']))]

    # print('exito')
    # acomodar resultados en columnas
    p_datos_ce['ho'] = [d_reaccion[j]['ho'] for j in range(0, len(p_datos_ce['timestamp']))]
    p_datos_ce['ol'] = [d_reaccion[j]['ol'] for j in range(0, len(p_datos_ce['timestamp']))]
    p_datos_ce['co'] = [d_reaccion[j]['co'] for j in range(0, len(p_datos_ce['timestamp']))]

    # Para cada indicador encontrar el escenario preponderante, después dividir en 4 grupos iguales los datos
    # hacer 4 grupos para prueba de anova, luego aplicar anova y regresar los indicadores en los que
    # se acepta la H0: "no hay diferencia significativa entre reacciones" del precio ante distintos escen.

    p_datos_ce = f_escenario(p0_datos=p_datos_ce)
    indicadores = list(set(p_datos_ce['Name']))
    ocurrencias = list()

    for i in range(0, len(indicadores)):
        # print(i)
        datos_ind = p_datos_ce[p_datos_ce['Name'] == indicadores[i]]
        conteos = list(datos_ind['escenario'].value_counts())
        con_max = int(np.trunc(max(conteos)/4))
        arg_max = np.argmax(list(datos_ind['escenario'].value_counts()))
        esc_max = list(datos_ind['escenario'].value_counts().index)[arg_max]
        datos_ind = datos_ind[datos_ind['escenario'] == esc_max]
        datos_ind = datos_ind.tail(con_max*4)
        datos_ind['sub_escenario'] = [ele for ele in [1, 2, 3, 4] for _ in range(con_max)]

        # crear cuadro de datos para modelo ANOVA
        df_data_anova = datos_ind[['escenario', 'co', 'sub_escenario']]
        # ajustar modelo lineal para (high - open)
        modelo = ols('co ~ C(sub_escenario)', data=df_data_anova).fit()
        anova_table = sm.stats.anova_lm(modelo, typ=2)
        res = anova_table['PR(>F)'][0]
        if res > 0.05:
            ocurrencias.append({'indicador': indicadores[i], 'escenario': esc_max,
                                'anova': anova_table})

    return pd.DataFrame({'indicador': [ocurrencias[i]['indicador'] for i in range(0, len(ocurrencias))],
                         'escenario': [ocurrencias[i]['escenario'] for i in range(0, len(ocurrencias))]})

# -- --------------------------------------- FUNCION: Generacion de variables ENDOGENAS series de tiempo -- #
# -- ------------------------------------------------------------------------------------ Version manual -- #

def f_features_end(p_datos):
    """
    :param p_datos: pd.DataFrae : dataframe con 5 columnas 'timestamp', 'open', 'high', 'low', 'close'
        :return: r_features : dataframe con 5 columnas, nombres cohercionados + Features generados

    # Debuging
    p_datos = df_precios
    p_datos = pd.DataFrame({''timestamp': {}, 'open': np.random.normal(1.1400, 0.0050, 20).
                                              'high': np.random.normal(1.1400, 0.0050, 20),
                                              'low': np.random.normal(1.1400, 0.0050, 20),
                                              'close': np.random.normal(1.1400, 0.0050, 20)})
    """

    datos = p_datos
    datos.columns = ['timestamp', 'open', 'high', 'low', 'close']

    cols = list(datos.columns)[1:]
    datos[cols] = datos[cols].apply(pd.to_numeric, errors='coerce')

    # formato columna timestamp como 'datetime'
    datos['timestamp'] = pd.to_datetime(datos['timestamp'])
    # datos['timestamp'] = datos['timestamp'].dt.tz_localize('UTC')

    # rendimiento logaritmico de ventana 1
    datos['logrend'] = np.log(datos['close']/datos['close'].shift(1)).dropna()

    # pips descontados al cierre
    datos['co'] = (datos['close']-datos['open'])*10000

    # pips descontados alcistas
    datos['ho'] = (datos['high'] - datos['open'])*10000

    # pips descontados bajistas
    datos['ol'] = (datos['open'] - datos['low'])*10000

    # pips descontados en total (medida de volatilidad)
    datos['hl'] = (datos['high'] - datos['low'])*10000

    # funciones de ACF y PACF para determinar ancho de ventana historica
    data_acf = acf(datos['logrend'].dropna(), nlags=12, fft=True)
    data_pac = pacf(datos['logrend'].dropna(), nlags=12)
    sig = round(1.96/np.sqrt(len(datos['logrend'])), 4)

    # componentes AR y MA
    maxs = list(set(list(np.where((data_pac > sig) | (data_pac < -sig))[0]) +
                    list(np.where((data_acf > sig) | (data_acf < -sig))[0])))
    # encontrar la componente maxima como indicativo de informacion historica autorelacionada
    max_n = maxs[np.argmax(maxs)]

    # condicion arbitraria: 5 resagos minimos para calcular variables moviles
    if max_n <= 2:
        max_n = 5

    # ciclo para calcular N features con logica de "Ventanas de tamaño n"
    for n in range(0, max_n):

        # resago n de ho
        datos['lag_ho_' + str(n + 1)] = np.log(datos['ho'].shift(n + 1))

        # resago n de ol
        datos['lag_ol_' + str(n + 1)] = np.log(datos['ol'].shift(n + 1))

        # promedio movil de ventana n
        datos['ma_ol_' + str(n + 2)] = datos['ol'].rolling(n + 2).mean()

        # promedio movil de ventana n
        datos['ma_ho_' + str(n + 2)] = datos['ho'].rolling(n + 2).mean()

    # asignar timestamp como index
    datos.index = pd.to_datetime(datos['timestamp'])
    # quitar columnas no necesarias para modelos de ML
    datos = datos.drop(['timestamp', 'open', 'high', 'low', 'close', 'hl', 'logrend'], axis=1)
    # borrar columnas donde exista solo NAs
    r_features = datos.dropna(axis='columns', how='all')
    # borrar renglones donde exista algun NA
    r_features = r_features.dropna(axis='rows')
    # convertir a numeros tipo float las columnas
    r_features.iloc[:, 1:] = r_features.iloc[:, 1:].astype(float)
    # estandarizacion de todas las variables independientes
    lista = r_features[list(r_features.columns[1:])]
    r_features[list(r_features.columns[1:])] = StandardScaler().fit_transform(lista)

    return r_features

# -- ---------------------------------------------------------------------- FUNCION: Ajustar RLM a datos -- #
# -- ---------------------------------------------------------------------- ---------------------------- -- #

def f_rlm(p_datos, p_y):
    """
    :param p_datos: pd.DataFrame : DataFrame con variable "y" (1era col), y n variables "x_n" (2:n)
    :param p_y : str : nombre de la columna a elegir como variable dependiente Y
    :return:
    p_datos = df_datos
    """

    datos = p_datos

    # Reacomodar los datos como arreglos
    y_multiple = np.array(datos[p_y])
    x_multiple = np.array(datos.iloc[:, 1:])

    # datos para entrenamiento y prueba
    train_x, test_x, train_y, test_y = train_test_split(x_multiple, y_multiple, test_size=0.2, shuffle=False)

    # Agregar interceptos a X en entrenamiento y prueba
    train_x_betha = sm.add_constant(train_x)
    test_x_betha = sm.add_constant(test_x)

    # Modelo ajustado (entrenamiento)
    modelo_train = sm.OLS(train_y, train_x_betha)
    # Resultados de ajuste de modelo (entrenamiento)
    modelo_fit_train = modelo_train.fit()

    # Modelo ajustado (prueba)
    modelo_test = sm.OLS(test_y, test_x_betha)
    # Resultados de ajuste de modelo (prueba)
    modelo_fit_test = modelo_test.fit()

    # -- -------------------------------------------------------------------- Con datos de ENTRENAMIENTO -- #
    # modelo completo resultante
    r_train_modelo = modelo_fit_train
    # DataFrame con nombre de parametros segun dataset, nombre de parametros y pvalues segun modelo
    r_df_train = pd.DataFrame({'df_params': ['intercepto'] + list(datos.columns[1:]),
                               'm_params': r_train_modelo.model.data.param_names,
                               'pv_params': r_train_modelo.pvalues})
    # valor de AIC del modelo
    r_train_aic = r_train_modelo.aic
    # valor de BIC del modelo
    r_train_bic = r_train_modelo.bic

    # -- --------------------------------------------------------------------------- Con datos de PRUEBA -- #
    # modelo completo resultante
    r_test_modelo = modelo_fit_test
    # DataFrame con nombre de parametros segun dataset, nombre de parametros y pvalues segun modelo
    r_df_test = pd.DataFrame({'df_params': ['intercepto'] + list(datos.columns[1:]),
                              'm_params': r_test_modelo.model.data.param_names,
                              'pv_params': r_test_modelo.pvalues})
    # valor de AIC del modelo
    r_test_aic = r_test_modelo.aic
    # valor de BIC del modelo
    r_test_bic = r_test_modelo.bic

    # tabla de resultados periodo de entrenamiento
    r_df_pred_train = pd.DataFrame({'y': train_y, 'y_ajustada': modelo_fit_train.predict()})
    # tabla de resultados periodo de prueba
    r_df_pred_test = pd.DataFrame({'y': test_y, 'y_ajustada': modelo_fit_test.predict()})
    # Errores cuadraticos medios para entrenamiento y prueba
    mse_train = round(mean_squared_error(r_df_pred_train['y'], r_df_pred_train['y_ajustada']), 2)
    mse_test = round(mean_squared_error(r_df_pred_test['y'], r_df_pred_test['y_ajustada']), 2)

    # -- --------------------------------------------------------------------------------- Datos finales -- #
    df_rlm = pd.DataFrame({'conjunto': ['train', 'test'],
                           'aic': [round(r_train_aic, 2), round(r_test_aic, 2)],
                           'bic': [round(r_train_bic, 2), round(r_test_bic, 2)],
                           'params': [len(r_df_train), len(r_df_test)],
                           'mse': [mse_train, mse_test],
                           'r2': [round(r_train_modelo.rsquared, 4), round(r_test_modelo.rsquared, 4)]})

    return df_rlm

# -- ---------------------------------------------------------------------- FUNCION: Aplicar PCA a datos -- #
# -- ---------------------------------------------------------------------- ---------------------------- -- #

def f_pca(p_datos, p_exp):
    """
    :param p_datos:
    :param p_exp:
    :return:

    p_datos = df_datos
    p_exp = .90
    """
    datos = p_datos

    pca = PCA(n_components=10)
    datos_pca = datos.iloc[:, 1:]
    pca.fit(datos_pca)
    # Calcular los vectores y valores propios de la martiz de covarianza
    w, v = np.linalg.eig(pca.get_covariance())
    # ordenar los valores de mayor a menor
    indx = np.argsort(w)[::-1]
    # calcular el procentaje de varianza en cada componente
    porcentaje = w[indx] / np.sum(w)
    # calcular el porcentaje acumulado de los componentes
    porcent_acum = np.cumsum(porcentaje)
    # encontrar las componentes necesarias para lograr explicar el 90% de variabilidad
    pca_90 = np.where(porcent_acum > p_exp)[0][0] + 1

    pca = PCA(n_components=pca_90)
    datos_pca = datos.iloc[:, 1:]
    df1 = datos.iloc[:, 0]
    pca.fit(datos_pca)
    df2 = pd.DataFrame(pca.transform(datos_pca))

    df1.reset_index(drop=True, inplace=True)
    df2.reset_index(drop=True, inplace=True)

    r_datos_pca = pd.concat([df1, df2], axis=1)
    r_datos_pca.index = datos_pca.index

    # Renombrar columnas
    r_datos_pca.columns = ['pca_y'] + ['pca_x_' + str(i) for i in range(0, pca_90)]

    return r_datos_pca

ext: py

reason: import numpy,import statsmodels,from statsmodels