# -*- coding: utf-8 -*-

import numpy as np
from ..data import Data, DataSamples
from ..model import LogisticRegressionModel
from .._utils import color_background
import matplotlib.pyplot as plt
import pandas as pd
from abc import abstractmethod
import copy
import gc
from sklearn.linear_model import LinearRegression, LogisticRegression
from itertools import combinations

plt.rc('font', family='Verdana')
plt.style.use('seaborn-darkgrid')
pd.set_option('display.precision', 3)

gc.enable()

class WeightedScore:
    '''
    Interface class for weighing several scores or features.

    '''
    def __init__(self):
        self.stats = pd.DataFrame()
        self.weights = {}
        self.reg = None

    @abstractmethod
    def fit(self):
        pass

    @abstractmethod
    def predict(self):
        pass

    @abstractmethod
    def to_sas(self, filename):
        pass

class BotWeightedScore(WeightedScore):
    '''
    For different scores weighting, including scores of outer vendors and data sources.
    Implements Yury Botnikov's algorithm for scores weighting with scores for missing values and protection against errors of integrations with outer production services.
    '''
    def __init__(self, root_score, outer_scores, target):
        '''
        Initialization

        Parameters
        ------------
        root_score: score considered the main, 'internal'
        outer_scores: list of scores to weight (except root_score)
        target: target for binary classification
        '''
        self.root_score = root_score
        if not isinstance(self.root_score, str):
            raise TypeError('Root_score should be a string! Only one root_score is allowed. Good luck.')
        self.target = target
        self.outer_scores = outer_scores
        self.scores_na = {}
        self.means = {}
        self.stds = {}
        self.coefs = {}
        self.stats = pd.DataFrame({'scores' : [self.root_score] + self.outer_scores})

    def na_scores(self, dataset, random_state = 42):
        '''
        TECH

        Calculates scores for missings in outer_scores

        Parameters
        -----------
        dataset: a train dataset, pandas.DataFrame
        random_state: parameter for LogisticRegression
        '''
        reg = LogisticRegression(random_state = random_state, C = 10000)
        reg.fit(dataset[[self.root_score] + self.outer_scores + [score + '_no_hit' for score in self.outer_scores]].fillna(0), dataset[self.target])

        n = len(self.outer_scores)
        self.stats['coefs_na_scores'] = reg.coef_[0][:n + 1]
        self.stats['coefs_na_no_hit'] = [np.nan] + list(reg.coef_[0][n + 1:])
        self.stats['scores_na'] = self.stats['coefs_na_scores']/self.stats['coefs_na_no_hit']

        for i, score in enumerate(self.outer_scores):
            self.scores_na[score] = reg.coef_[0][i + n + 1]/reg.coef_[0][i + 1]

    def score_hist(self, score, dataset, round_val = 1):
        '''
        Plots a histogram of score values wiht scores for missings.

        Parameters
        -----------
        score: str, name of score to plot
        dataset: pandas.DataFrame with score
        round_val: int, rounding parameter
        '''
        fig, ax = plt.subplots()
        fig.set_figwidth(12)
        n, bins, patches = ax.hist(list(dataset[score].dropna()), bins = 40)
        if score in self.scores_na:
            patches[max(np.where(bins < self.scores_na[score])[0])].set_fc('r')
            patches[max(np.where(bins < self.scores_na[score])[0])].set_label('Score for missing value')
            ax.legend()
        ax.set_title(score)
        plt.xticks(bins, np.around(bins, round_val), rotation = 17)
        plt.show()

    def set_scores_na(self, scores_na, verbose = True):
        '''
        Setter for scores_na
        '''
        self.scores_na = scores_na
        self.stats['scores_na'] = [v for k, v in self.scores_na.items()]

    def calc_coefs(self, dataset, random_state = 42):
        '''
        TECH

        Calculates normalized regression coefficients for root score and outer scores.

        Parameters
        ------------
        dataset: pandas.DataFrame to process
        '''
        reg = LogisticRegression(random_state = random_state, C = 10000)
        reg.fit(dataset[[self.root_score] + self.outer_scores], dataset[self.target])
        self.stats['coefs'] = reg.coef_[0]
        self.coefs = {s: reg.coef_[0][i]/reg.coef_[0][0] for i, s in enumerate([self.root_score] + self.outer_scores)}
        self.stats['coefs_norm'] = list(self.coefs.values())

    def calc_moments(self, dataset):
        '''
        TECH

        Calculates mean and std for root score and outer scores

        Parameters
        -----------
        dataset: pandas.DataFrame with data
        '''
        for score in [self.root_score] + self.outer_scores:
            self.means[score] = dataset[score].mean()
            self.stds[score] = dataset[score].std()
        self.stats['means'] = list(self.means.values())
        self.stats['stds'] = list(self.stds.values())

    def fit(self, dataset, verbose = True, calc_na = True, random_state = 42):
        '''
        Calculates weights and "scores for missings"

        Parameters
        ------------
        dataset: Data or pandas.DataFrame, dataset with scores, including root_score an outer_scores
        random_state: parameter for LogisticRegression
        verbose: boolean,if True - the fitting process is visualized
        calc_na: boolean, if True - values for missings are calculated automatically
        '''

        train = dataset.dataframe[[self.target, self.root_score] + self.outer_scores].copy() if isinstance(dataset, Data) else dataset[[self.target, self.root_score] + self.outer_scores].copy()
        if verbose:
            print ('Calculation started...')

        # Step 1
        if calc_na:
            # Scores for missings
            if train.isnull().sum().sum() > 0:
                for score in self.outer_scores:
                    train[score + '_no_hit'] = train[score].isnull()*1
                self.na_scores(train, random_state)
            else:
                self.scores_na = {score : train[score].median() for score in self.outer_scores}
        elif len(self.scores_na) < len(self.outer_scores):
            raise ValueError('All the outer scores should present in scores_na')

        if verbose:
            print ('Scores for missings:', self.scores_na)
            for score in self.outer_scores:
                self.score_hist(score, train)

        # Step 2
        for score, val in self.scores_na.items():
            train[score + '_hit'] = 1*(train[score].isnull() == False)
            train[score].fillna(val, inplace = True)
        self.calc_coefs(train, random_state)
        if verbose:
            print ('Normalized coefficients:', self.coefs)

        # Step 3
        self.calc_moments(train)
        if verbose:
            print ('Mean values of scores:', self.means)
            print ('Standard deviations of scores:', self.stds)

    def predict(self, dataset, score_name = 'weighted_score', floor = False, errors = False):
        '''
        Calculates final weighted score from root_score and outer_scores.

        Parameters:
        -------------
        dataset: pandas.DataFrame or Data, dataset for processing
        score: str, final score's name
        floor: boolean, if True - np.floor is applied to the final score
        errors: dict of names of columns with errors in outer scores, e.g. if an integration with an outer source is broken or a score is invalid; columns with errors should
        contain 2 values: 1 - no error, 0 - error.
            Example:
                {'score_beeline_t' : 'score_beeeline_valid', 'score_MFM' : 'score_MFM_valid'}
        '''
        if errors:
            print ('Attention! {} should be 1 if the value is correct, and 0 if the value is damaged'.format(list(errors.values())))

        to_data = isinstance(dataset, Data)
        if isinstance(dataset, pd.DataFrame):
            df = copy.deepcopy(dataset)
            df0 = copy.deepcopy(dataset)
        elif to_data:
            df = copy.deepcopy(dataset.dataframe)
            df0 = copy.deepcopy(dataset.dataframe)
        else:
            raise TypeError('Wrong type of dataset! Bye-bye.')

        for score, val in self.scores_na.items():
            df[score].fillna(val, inplace = True)

        nom = df[self.root_score].copy()
        denom = pd.Series(np.ones(df.shape[0]))
        for score in self.outer_scores:
            nom += (df[score] - self.means[score])*(self.coefs[score] if not errors else self.coefs[score]*dataset[errors[score]])
            denom += self.coefs[score]**2*self.stds[score]**2/self.stds[self.root_score]**2*(pd.Series(np.ones(df.shape[0])) if not errors else self.coefs[score]*dataset[errors[score]])

        denom = np.sqrt(denom)

        df0[score_name] = (nom - self.means[self.root_score])/denom + self.means[self.root_score]
        if floor:
            df0[score_name] = np.floor(df0[score_name])

        if to_data:
            return Data(df0, target = dataset.target, features = dataset.features, weights = dataset.weights)
        else:
            return df0

    def to_sas(self, filename, round_digits = 3, result_score_name = 'weighted_score', score_prefix = 'SCR_', flag_prefix = 'Flag_', root_score_name = None,
               outer_scores_names = None, verbose = True):
        '''
        SAS-code generation

        parameters
        ----------
        filename: str, name of the file to write into
        round_digits: int, parameter of rounding
        result_score_name: str, name of the resulting score in the SAS-code
        score_prefix: str, prefix for edited scores
        flag_prefix: str, prefix for flags of valid scores
        root_score_name: str, name of the root score in the script
        outer_scores_names: dict, {outer_score : outer_score_name} if scores' names differ from outer_scores
        '''
        if root_score_name is None:
            root_score_name = self.root_score

        if outer_scores_names is None:
            outer_scores_names = {n : n for n in self.outer_scores}

        with open(filename, 'w') as f:
            for i, score in enumerate(self.outer_scores):
                row = 'if missing(' + outer_scores_names[score] + ') then ' + score_prefix + outer_scores_names[score] + ' = ' + str(round(self.scores_na[score] - self.means[score], round_digits)) + ';\n'
                row += score_prefix + outer_scores_names[score] + ' = ' + outer_scores_names[score] + ' - ' + str(round(self.means[score], round_digits)) + ';\n'
                f.write(row)
                if verbose:
                    print (row)
            f.write('/***************************************************************/\n')
            f.write('/*********************** Final score ***************************/\n')
            f.write('/***************************************************************/\n')
            row = 'nom = ' + root_score_name
            for score in self.outer_scores:
                row += ' + ' + flag_prefix + outer_scores_names[score] + ' * ' + score_prefix + outer_scores_names[score] + ' * ' + str(round(self.coefs[score], round_digits))
            row += ';\n'
            f.write(row)
            if verbose:
                print (row)

            row = 'denom = sqrt(1'
            for score in self.outer_scores:
                row += ' + ' + flag_prefix + outer_scores_names[score] + '*' + str(round(self.coefs[score]**2*self.stds[score]**2/self.stds[self.root_score]**2, round_digits))
            row += ');\n'

            f.write(row)
            if verbose:
                print (row)

            row = result_score_name + ' = floor((nom - ' + str(round(self.means[self.root_score], round_digits)) + ') / denom + '  + str(round(self.means[self.root_score], round_digits)) + ');\n'
            f.write(row)
            if verbose:
                print (row)

            f.close()

class VanillaWeightedScore(WeightedScore):
    '''
    Implements plain weightening: firstly, each score is divided into 2 zones: zone with missings and zone without missings. After that, for each combination of the scores'
    zones separate weigths are calculated (for non-missing scores).
    E.g. (score1, score2, score3) ->
        -> f(score1, score2, score3) = (score1 is missing)*((score2 is missing)*f1(score3)
                                                            + (score3 is missing)*f2(score2)
                                                            + (score2 is not missing and score3 is not missing)*f3(score2, score3)
                                                            )
                                      + (score2 is missing)*((score3 is missing)*f4(score1)
                                                            + (score1 is not missing and score3 is not missing)*f5(score1, score3)
                                                            )
                                      + (score3 is missing)*(score1 is not missing and score2 is not missing)*f6(score1, score2)
    '''
    def fit(self, dataset, scores = None, target = None, task_type = 'binary', random_state = 42):
        '''
        Calculates weights for scores

        Parameters
        -----------
        dataset: pandas.DataFrame or data.Data object with scores and binary target
        scores: (optional) list of scored to weighing
        target: (optional) name of target
        task_type: (optional) 'binary' for binary classification task or 'linear' for linear regression task
        random_state: random_state
        '''
        if isinstance(dataset, Data):
            if scores is None:
                scores = dataset.features
            if target is None:
                target = dataset.target
            df = copy.deepcopy(dataset.dataframe)
        elif isinstance(dataset, pd.DataFrame):
            df = copy.deepcopy(dataset)
        else:
            raise TypeError('Found {} where pandas.DataFrame or scorekit.data.Data was expected! Good luck!'.format(type(dataset)))

        if scores is None or target is None or target not in dataset.columns or sum([i in dataset.columns for i in scores]) < len(scores):
            raise ValueError('Please set correct scores and target! Good luck!')

        df = copy.deepcopy(df[[target] + scores])
        if task_type == 'binary':
            self.reg = LogisticRegression(random_state = random_state, C = 10000)
        elif task_type == 'linear':
            self.reg = LinearRegression()
        else:
            raise ValueError('Wrong task_type! Bye-bye.')

        # find scores with missings
        scores_with_nan = list(df.isnull().sum()[df.isnull().sum() > 0].index)

        # make combinations of scores
        ranges = []
        for i in range(1, len(scores_with_nan) + 1):
            for subset in combinations(scores_with_nan, i):
                ranges.append(subset)

        self.zones = dict((ind, list(v)) for ind, v in enumerate(ranges))
        del ranges

        for zone, zone_scores in self.zones.items():
            self.weights[zone] = self.fit_zone(zone_scores, df, scores, target)

    def fit_zone(self, zone_scores, df, scores, target):
        '''
        TECH

        Finds coefs for 1 zone

        Parameters
        -----------
        zone_scores: scores that define zone (from self.zones)
        df: pd.DataFrame to fit on
        scores: list of all the scores used
        target: target name

        Returns
        -----------
        result: dict with coefficients for each score and intercept
        '''
        df_fit = df[df[zone_scores].isnull().sum(axis = 1) > 0].drop(zone_scores, 1).dropna()
        if df_fit.shape[0] > 0:
            try:
                self.reg.fit(df_fit.drop(target, 1), df_fit[target])
                result = {'intercept': self.reg.intercept_[0]}
                for i, f in enumerate(df_fit.drop(target, 1).columns):
                    result[f] = self.reg.coef_[0][i]
            except Exception:
                result = {'intercept': df_fit[target].mean()}
                for i, f in enumerate(df_fit.drop(target, 1).columns):
                    result[f] = 0
            return result

    def predict(self, dataset, weighted_name = 'score_weighted'):
        '''
        Calculates weighted score for the dataset

        Parameters
        -----------
        dataset: a scorekit.data.Data or pandas.DataFrame object with scores to weight
        weighted_name: name for column with weighted score

        Returns
        ----------
        an object ot type(dataset) with weighted score
        '''
        try:
            if isinstance(dataset, Data):
                df = copy.deepcopy(dataset.dataframe)
            elif isinstance(dataset, pd.DataFrame):
                df = copy.deepcopy(dataset)
            else:
                raise TypeError('Wrong type of dataset! Try DataFrame or Data. Good luck >:-]')
        except Exception:
            print ('Initialization failed!')


"""
Palindrome Permutation
https://leetcode.com/problems/palindrome-permutation/

Time O(n)
Space O(n)
"""
from collections import Counter

class Solution:
    def canPermutePalindrome(self, s: str) -> bool:
        charFreq = Counter(s)
        oddSeen = False
        for count in charFreq.values():
            if count % 2 != 0:
                if oddSeen:
                    return False
                oddSeen = True
        return True


import json
import time
import pickle
import logging
import datetime
from indra.databases import ndex_client
import indra.tools.assemble_corpus as ac
from indra.literature import pubmed_client, elsevier_client
from indra.assemblers.cx import CxAssembler
from indra.assemblers.pysb import PysbAssembler
from indra.assemblers.pybel import PybelAssembler
from indra.assemblers.indranet import IndraNetAssembler
from indra.mechlinker import MechLinker
from indra.preassembler.hierarchy_manager import get_wm_hierarchies
from indra.preassembler import Preassembler
from indra.belief.wm_scorer import get_eidos_scorer
from indra.statements import Event, Association
from emmaa.priors import SearchTerm
from emmaa.readers.aws_reader import read_pmid_search_terms
from emmaa.readers.db_client_reader import read_db_pmid_search_terms
from emmaa.readers.elsevier_eidos_reader import \
    read_elsevier_eidos_search_terms
from emmaa.util import make_date_str, find_latest_s3_file, get_s3_client

logger = logging.getLogger(__name__)

class EmmaaModel(object):
    """"Represents an EMMAA model.

    Parameters
    ----------
    name : str
        The name of the model.
    config : dict
        A configuration dict that is typically loaded from a YAML file.

    Attributes
    ----------
    name : str
        A string containing the name of the model
    stmts : list[emmaa.EmmaaStatement]
        A list of EmmaaStatement objects representing the model
    assembly_config : dict
        Configurations for assembling the model.
    test_config : dict
        Configurations for running tests on the model.
    reading_config : dict
        Configurations for reading the content.
    search_terms : list[emmaa.priors.SearchTerm]
        A list of SearchTerm objects containing the search terms used in the
        model.
    ndex_network : str
        The identifier of the NDEx network corresponding to the model.
    assembled_stmts : list[indra.statements.Statement]
        A list of assembled INDRA Statements
    """
    def __init__(self, name, config):
        self.name = name
        self.stmts = []
        self.assembly_config = {}
        self.test_config = {}
        self.reading_config = {}
        self.search_terms = []
        self.ndex_network = None
        self._load_config(config)
        self.assembled_stmts = []

    def add_statements(self, stmts):
        """"Add a set of EMMAA Statements to the model

        Parameters
        ----------
        stmts : list[emmaa.EmmaaStatement]
            A list of EMMAA Statements to add to the model
        """
        self.stmts += stmts

    def get_indra_stmts(self):
        """Return the INDRA Statements contained in the model.

        Returns
        -------
        list[indra.statements.Statement]
            The list of INDRA Statements that are extracted from the EMMAA
            Statements.
        """
        return [es.stmt for es in self.stmts]

    def _load_config(self, config):
        self.search_terms = [SearchTerm.from_json(s) for s in
                             config['search_terms']]
        if 'ndex' in config:
            self.ndex_network = config['ndex']['network']
        else:
            self.ndex_network = None
        if 'reading' in config:
            self.reading_config = config['reading']
        if 'assembly' in config:
            self.assembly_config = config['assembly']
        if 'test' in config:
            self.test_config = config['test']

    def search_literature(self, date_limit=None):
        """Search for the model's search terms in the literature.

        Parameters
        ----------
        date_limit : Optional[int]
            The number of days to search back from today.

        Returns
        -------
        ids_to_terms : dict
            A dict representing all the literature source IDs (e.g.,
            PMIDs or PIIS) returned by the searches as keys,
            and the search terms for which the given ID was produced as
            values.
        """
        lit_source = self.reading_config.get('literature_source', 'pubmed')
        if lit_source == 'pubmed':
            terms_to_ids = self.search_pubmed(self.search_terms, date_limit)
        elif lit_source == 'elsevier':
            terms_to_ids = self.search_elsevier(self.search_terms, date_limit)
        else:
            raise ValueError('Unknown literature source: %s' % lit_source)
        ids_to_terms = {}
        for term, ids in terms_to_ids.items():
            for id in ids:
                try:
                    ids_to_terms[id].append(term)
                except KeyError:
                    ids_to_terms[id] = [term]
        return ids_to_terms

    @staticmethod
    def search_pubmed(search_terms, date_limit):
        """Search PubMed for given search terms.

        Parameters
        ----------
        search_terms : list[emmaa.priors.SearchTerm]
            A list of SearchTerm objects to search PubMed for.
        date_limit : int
            The number of days to search back from today.

        Returns
        -------
        terms_to_pmids : dict
            A dict representing given search terms as keys and PMIDs returned
            by searches as values.
        """
        terms_to_pmids = {}
        for term in search_terms:
            pmids = pubmed_client.get_ids(term.search_term, reldate=date_limit)
            logger.info(f'{len(pmids)} PMIDs found for {term.search_term}')
            terms_to_pmids[term] = pmids
            time.sleep(1)
        return terms_to_pmids

    @staticmethod
    def search_elsevier(search_terms, date_limit):
        """Search Elsevier for given search terms.

        Parameters
        ----------
        search_terms : list[emmaa.priors.SearchTerm]
            A list of SearchTerm objects to search PubMed for.
        date_limit : int
            The number of days to search back from today.

        Returns
        -------
        terms_to_piis : dict
            A dict representing given search terms as keys and PIIs returned
            by searches as values.
        """
        start_date = (
            datetime.datetime.utcnow() - datetime.timedelta(days=date_limit))
        start_date = start_date.isoformat(timespec='seconds') + 'Z'
        terms_to_piis = {}
        for term in search_terms:
            # NOTE for now limiting the search to only 5 PIIs
            piis = elsevier_client.get_piis_for_date(
                term.search_term, loaded_after=start_date)[:5]
            logger.info(f'{len(piis)} PIIs found for {term.search_term}')
            terms_to_piis[term] = piis
            time.sleep(1)
        return terms_to_piis

    def get_new_readings(self, date_limit=10):
        """Search new literature, read, and add to model statements"""
        reader = self.reading_config.get('reader', 'indra_db')
        ids_to_terms = self.search_literature(date_limit=date_limit)
        if reader == 'aws':
            estmts = read_pmid_search_terms(ids_to_terms)
        elif reader == 'indra_db':
            estmts = read_db_pmid_search_terms(ids_to_terms)
        elif reader == 'elsevier_eidos':
            estmts = read_elsevier_eidos_search_terms(ids_to_terms)
        else:
            raise ValueError('Unknown reader: %s' % reader)

        self.extend_unique(estmts)

    def extend_unique(self, estmts):
        """Extend model statements only if it is not already there."""
        source_hashes = {est.stmt.get_hash(shallow=False, refresh=True)
                         for est in self.stmts}
        for estmt in estmts:
            if estmt.stmt.get_hash(shallow=False, refresh=True) not in \
                    source_hashes:
                self.stmts.append(estmt)

    def eliminate_copies(self):
        """Filter out exact copies of the same Statement."""
        logger.info('Starting with %d raw EmmaaStatements' % len(self.stmts))
        self.stmts = list({estmt.stmt.get_hash(shallow=False, refresh=True):
                           estmt for estmt in self.stmts}.values())
        logger.info(('Continuing with %d raw EmmaaStatements'
                     ' that are not exact copies') % len(self.stmts))

    def run_assembly(self):
        """Run INDRA's assembly pipeline on the Statements."""
        self.eliminate_copies()
        stmts = self.get_indra_stmts()
        stmts = self.filter_event_association(stmts)
        stmts = ac.filter_no_hypothesis(stmts)
        if not self.assembly_config.get('skip_map_grounding'):
            stmts = ac.map_grounding(stmts)
        if self.assembly_config.get('standardize_names'):
            ac.standardize_names_groundings(stmts)
        if self.assembly_config.get('filter_ungrounded'):
            score_threshold = self.assembly_config.get('score_threshold')
            stmts = ac.filter_grounded_only(
                stmts, score_threshold=score_threshold)
        if self.assembly_config.get('merge_groundings'):
            stmts = ac.merge_groundings(stmts)
        if self.assembly_config.get('merge_deltas'):
            stmts = ac.merge_deltas(stmts)
        relevance_policy = self.assembly_config.get('filter_relevance')
        if relevance_policy:
            stmts = self.filter_relevance(stmts, relevance_policy)
        if not self.assembly_config.get('skip_filter_human'):
            stmts = ac.filter_human_only(stmts)
        if not self.assembly_config.get('skip_map_sequence'):
            stmts = ac.map_sequence(stmts)
        # Use WM hierarchies and belief scorer for WM preassembly
        preassembly_mode = self.assembly_config.get('preassembly_mode')
        if preassembly_mode == 'wm':
            hierarchies = get_wm_hierarchies()
            belief_scorer = get_eidos_scorer()
            stmts = ac.run_preassembly(
                stmts, return_toplevel=False, belief_scorer=belief_scorer,
                hierarchies=hierarchies)
        else:
            stmts = ac.run_preassembly(stmts, return_toplevel=False)
        belief_cutoff = self.assembly_config.get('belief_cutoff')
        if belief_cutoff is not None:
            stmts = ac.filter_belief(stmts, belief_cutoff)
        stmts = ac.filter_top_level(stmts)

        if self.assembly_config.get('filter_direct'):
            stmts = ac.filter_direct(stmts)
            stmts = ac.filter_enzyme_kinase(stmts)
            stmts = ac.filter_mod_nokinase(stmts)
            stmts = ac.filter_transcription_factor(stmts)

        if self.assembly_config.get('mechanism_linking'):
            ml = MechLinker(stmts)
            ml.gather_explicit_activities()
            ml.reduce_activities()
            ml.gather_modifications()
            ml.reduce_modifications()
            ml.gather_explicit_activities()
            ml.replace_activations()
            ml.require_active_forms()
            stmts = ml.statements

        self.assembled_stmts = stmts

    def filter_event_association(self, stmts):
        """Filter a list of Statements to exclude Events and Associations."""
        logger.info('Filtering Events and Associations.')
        stmts = [stmt for stmt in stmts if (
            (not isinstance(stmt, Event)) and
            (not isinstance(stmt, Association)))]
        logger.info('%d statements after filter...' % len(stmts))
        return stmts

    def filter_relevance(self, stmts, policy=None):
        """Filter a list of Statements to ones matching a search term."""
        logger.info('Filtering %d statements for relevance...' % len(stmts))
        stmts_out = []
        stnames = {s.name for s in self.search_terms}
        for stmt in stmts:
            agnames = {a.name for a in stmt.agent_list() if a is not None}
            if policy == 'prior_one' and (agnames & stnames):
                stmts_out.append(stmt)
            elif policy == 'prior_all' and agnames.issubset(stnames):
                stmts_out.append(stmt)
            elif policy is None:
                stmts_out.append(stmt)
        logger.info('%d statements after filter...' % len(stmts_out))
        return stmts_out

    def update_to_ndex(self):
        """Update assembled model as CX on NDEx, updates existing network."""
        if not self.assembled_stmts:
            self.run_assembly()
        cxa = CxAssembler(self.assembled_stmts, network_name=self.name)
        cxa.make_model()
        cx_str = cxa.print_cx()
        ndex_client.update_network(cx_str, self.ndex_network)

    def upload_to_ndex(self):
        """Upload the assembled model as CX to NDEx, creates new network."""
        if not self.assembled_stmts:
            self.run_assembly()
        cxa = CxAssembler(self.assembled_stmts, network_name=self.name)
        cxa.make_model()
        model_uuid = cxa.upload_model()
        self.ndex_network = model_uuid
        return model_uuid

    def save_to_s3(self):
        """Dump the model state to S3."""
        date_str = make_date_str()
        fname = f'models/{self.name}/model_{date_str}'
        client = get_s3_client(unsigned=False)
        # Dump as pickle
        client.put_object(Body=pickle.dumps(self.stmts), Bucket='emmaa',
                          Key=fname+'.pkl')
        # Dump as json
        client.put_object(Body=str.encode(json.dumps(self.to_json()),
                                          encoding='utf8'),
                          Bucket='emmaa', Key=fname+'.json')

    @classmethod
    def load_from_s3(klass, model_name):
        """Load the latest model state from S3.

        Parameters
        ----------
        model_name : str
            Name of model to load. This function expects the latest model
            to be found on S3 in the emmaa bucket with key
            'models/{model_name}/model_{date_string}', and the model config
            file at 'models/{model_name}/config.json'.

        Returns
        -------
        emmaa.model.EmmaaModel
            Latest instance of EmmaaModel with the given name, loaded from S3.
        """
        config = load_config_from_s3(model_name)
        stmts = load_stmts_from_s3(model_name)
        em = klass(model_name, config)
        em.stmts = stmts
        return em

    def get_entities(self):
        """Return a list of Agent objects that the model contains."""
        istmts = self.get_indra_stmts()
        agents = []
        for stmt in istmts:
            agents += [a for a in stmt.agent_list() if a is not None]
        return agents

    def get_assembled_entities(self):
        """Return a list of Agent objects that the assembled model contains."""
        if not self.assembled_stmts:
            self.run_assembly()
        agents = []
        for stmt in self.assembled_stmts:
            agents += [a for a in stmt.agent_list() if a is not None]
        return agents

    def assemble_pysb(self):
        """Assemble the model into PySB and return the assembled model."""
        if not self.assembled_stmts:
            self.run_assembly()
        pa = PysbAssembler()
        pa.add_statements(self.assembled_stmts)
        pysb_model = pa.make_model()
        return pysb_model

    def assemble_pybel(self):
        """Assemble the model into PyBEL and return the assembled model."""
        if not self.assembled_stmts:
            self.run_assembly()
        pba = PybelAssembler(self.assembled_stmts)
        pybel_model = pba.make_model()
        return pybel_model

    def assemble_signed_graph(self):
        """Assemble the model into signed graph and return the assembled graph."""
        if not self.assembled_stmts:
            self.run_assembly()
        ia = IndraNetAssembler(self.assembled_stmts)
        signed_graph = ia.make_model(graph_type='signed')
        return signed_graph

    def assemble_unsigned_graph(self):
        """Assemble the model into unsigned graph and return the assembled graph."""
        if not self.assembled_stmts:
            self.run_assembly()
        ia = IndraNetAssembler(self.assembled_stmts)
        unsigned_graph = ia.make_model(graph_type='digraph')
        return unsigned_graph

    def to_json(self):
        """Convert the model into a json dumpable dictionary"""
        json_output = {'name': self.name,
                       'ndex_network': self.ndex_network,
                       'search_terms': [st.to_json() for st
                                        in self.search_terms],
                       'stmts': [st.to_json() for st in self.stmts]}
        return json_output

    def __repr__(self):
        return "EmmaModel(%s, %d stmts, %d search terms)" % \
                   (self.name, len(self.stmts), len(self.search_terms))

def load_config_from_s3(model_name):
    """Return a JSON dict of config settings for a model from S3.

    Parameters
    ----------
    model_name : str
        The name of the model whose config should be loaded.

    Returns
    -------
    config : dict
        A JSON dictionary of the model configuration loaded from S3.
    """
    client = get_s3_client()
    base_key = f'models/{model_name}'
    config_key = f'{base_key}/config.json'
    logger.info(f'Loading model config from {config_key}')
    obj = client.get_object(Bucket='emmaa', Key=config_key)
    config = json.loads(obj['Body'].read().decode('utf8'))
    return config

def save_config_to_s3(model_name, config):
    """Upload config settings for a model to S3.

    Parameters
    ----------
    model_name : str
        The name of the model whose config should be saved to S3.
    config : dict
        A JSON dict of configurations for the model.
    """
    client = get_s3_client(unsigned=False)
    base_key = f'models/{model_name}'
    config_key = f'{base_key}/config.json'
    logger.info(f'Saving model config to {config_key}')
    client.put_object(Body=str.encode(json.dumps(config, indent=1),
                                      encoding='utf8'),
                      Bucket='emmaa', Key=config_key)

def load_stmts_from_s3(model_name):
    """Return the list of EMMAA Statements constituting the latest model.

    Parameters
    ----------
    model_name : str
        The name of the model whose config should be loaded.

    Returns
    -------
    stmts : list of emmaa.statements.EmmaaStatement
        The list of EMMAA Statements in the latest model version.
    """
    client = get_s3_client()
    base_key = f'models/{model_name}'
    latest_model_key = find_latest_s3_file('emmaa', f'{base_key}/model_',
                                           extension='.pkl')
    logger.info(f'Loading model state from {latest_model_key}')
    obj = client.get_object(Bucket='emmaa', Key=latest_model_key)
    stmts = pickle.loads(obj['Body'].read())
    return stmts


from django.shortcuts import render, HttpResponseRedirect
from django.contrib import messages
from .models import Clientgallery
from client.models import Client
from django.contrib.auth import authenticate, login, logout
from django.contrib.auth.decorators import login_required

# Create your views here.

@login_required(login_url='login')
def addclig(request):
    cli = Client.objects.all()
    if request.method == 'POST':
        cliid = request.POST['cliid']
        cligimg = request.FILES.getlist('cligimg')
        sta = request.POST['sta']
        for cligimg1 in cligimg:
            add = Clientgallery(cli_id_id=cliid, clig_img=cligimg1, status=sta)

            add.save()

        messages.success(request, 'Client Gallery Image Added Successfully', extra_tags='success')

        return HttpResponseRedirect('/cligly/manageclientgallery/')

        # msg = {'serr':'Client Gallery Image Added Successfully'}
        # return render(request, 'clientgallery/add_clig.html', {'err':msg, 'cli':cli, 'name': request.user})
    else:
        # msg = {'ferr':'Please Fill All Field Either Client Gallery Image Do Not Add.'}
        return render(request, 'clientgallery/add_clig.html', {'cli':cli, 'name': request.user})
    return render(request, 'clientgallery/add_clig.html', {'cat':cat, 'name': request.user})

@login_required(login_url='login')
def edtclig(request, eid):
    eclig = Clientgallery.objects.get(pk=eid)
    cli = Client.objects.all()
    return render(request, 'clientgallery/edit_clig.html', {'eclig':eclig, 'cli':cli, 'name': request.user})

@login_required(login_url='login')
def updclig(request, ucligid):
    if request.method == 'POST':
        try:
            if request.FILES['cligimg'] != 0:
                cliid = request.POST['cliid']
                cligimg = request.FILES['cligimg']
                sta = request.POST['sta']

                edt = Clientgallery.objects.get(clig_id = ucligid)

                edt.cli_id_id = cliid
                edt.clig_img = cligimg
                edt.status = sta

                edt.save()
        except:
            cliid = request.POST['cliid']
            sta = request.POST['sta']

            edt = Clientgallery.objects.get(clig_id = ucligid)

            edt.cli_id_id = cliid
            edt.clig_img = edt.clig_img
            edt.status = sta

            edt.save()

            messages.success(request, 'Client Gallery Image Updated Successfully', extra_tags='success')

            return HttpResponseRedirect('/cligly/manageclientgallery/')

        messages.success(request, 'Client Gallery Image Updated Successfully', extra_tags='success')

        return HttpResponseRedirect('/cligly/manageclientgallery/')

@login_required(login_url='login')
def delclig(request, did):
    de = Clientgallery.objects.get(pk=did)
    de.delete()
    messages.success(request, 'Client Gallery Image Deleted Successfully', extra_tags='danger')
    return HttpResponseRedirect('/cligly/manageclientgallery/')

@login_required(login_url='login')
def manclig(request):
    clig = Clientgallery.objects.all()
    return render(request, 'clientgallery/man_clig.html', {'clig': clig, 'name': request.user})


from dice import Pipeline
from dice.evaluation import Tracker
from dice.evaluation import PairEvaluator

from functools import reduce
import hyperopt
import pickle
import os

class BayesianTuner:

    def __init__(self, annotation_file, feature, *folders):
        self.annotation_file = annotation_file
        self.folders = folders
        def objective(args):
            trackers = list()
            for folder in self.folders:
                pipeline = Pipeline(
                    folder,
                    reduce(lambda x,y: dict(x, **y), (args, {"log": ""}))
                )
                pipeline.load()
                if feature == "evidence":
                    pipeline.step_detective()
                if feature == "confidence":
                    pipeline.step_assigner()
                tracker = Tracker()
                tracker.build(pipeline)
                trackers.append(tracker)
            PairEvaluator.FEATURE = feature
            return PairEvaluator(self.annotation_file, *trackers).evaluate()
        self.objective = objective
        self.choices = dict()

    def parse_space(self, args):
        space = dict()
        for arg in args.split(" "):
            param, rng = arg.split("=")
            if "-" in rng:
                low, high = list(map(float, rng.split("-")))
                space[param] = hyperopt.hp.uniform(param, low, high)
            elif "," in rng:
                values = list(map(float, rng.split(",")))
                space[param] = hyperopt.hp.choice(param, values)
                self.choices[param] = values
            else:
                space[param] = hyperopt.hp.choice(param, (float(rng),))
                self.choices[param] = [float(rng)]
        return space

    def optimize(self, args, max_evals=200):
        space = self.parse_space(args)
        if os.path.isfile("trials.pickle"):
            with open("trials.pickle", "rb") as file:
                trials = pickle.load(file)
        else:
            trials = hyperopt.Trials()
        best = hyperopt.fmin(
            self.objective,
            space,
            algo=hyperopt.tpe.suggest,
            max_evals=max_evals,
            trials=trials,
        )
        pickle.dump(trials, open("trials.pickle", "wb"))
        for param, values in self.choices.items():
            best[param] = values[best[param]]
        return best


from flask import *
from whitenoise import WhiteNoise
app = Flask(__name__)
app.wsi_app = WhiteNoise(app.wsgi_app, root='static/', prefix='static/', index_file="index.html", autorefresh=True)

@app.route('/', methods=['GET'])
def home():
  # return render_template("index.html")
  return "Jenna Russell: https://jjr265-info4310-hw2.herokuapp.com/static/index.html"

if __name__ == "__main__":
  app.debug = True
  app.run(threaded=True, port=5000)


# coding=utf-8
'''
Created on 2013-2-28

@author: lijie1
'''

import logging;
import os;
from datetime import date;
from lib import CMail;
from configs import Config;

class Log(object):
    def __init__(self):
        """Do nothing, by default."""

    @staticmethod
    def write(msg, sType = 'info'):

        sCurrentDir = os.getcwd();
        sLogDir = sCurrentDir + '/Logs/';
        if not os.path.exists(sLogDir):
            os.makedirs(sLogDir);

        today = date.today();
        sLogFile = '';
        if sType == 'info':
            sLogFile = 'Log_{0:04d}_{1:02d}_{2:02d}.log'.format(today.year, today.month, today.day);
        if sType == 'debug':
            sLogFile = 'Debug_{0:04d}_{1:02d}_{2:02d}.log'.format(today.year, today.month, today.day);
        if sType == 'error':
            sLogFile = 'Error_{0:04d}_{1:02d}_{2:02d}.log'.format(today.year, today.month, today.day);

        logging.basicConfig(level=logging.DEBUG,
                format   ='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s',
                datefmt  ='%Y-%m-%d %H:%M:%S',
                filename = sLogDir + sLogFile,
                filemode ='a');
        msg = msg.encode('gb2312');
        if sType == 'info':
            logging.info(msg);
        if sType == 'debug':
            logging.debug(msg);
#         if sType == 'error':
#             mail = CMail(True);
#             mail.sendMail(Config.YL_MAIL['server'], Config.YL_MAIL['from'], Config.YL_MAIL['to'], [], ' coolshow python task error', str(msg));
            logging.error(msg);


from .._compat import PY3

if PY3:
    import asyncio
else:
    import trollius as asyncio

from zope.testing import setupstack
from concurrent.futures import Future
import mock
from ZODB.POSException import ReadOnlyError
from ZODB.utils import maxtid, RLock

import collections
import logging
import struct
import unittest

from ..Exceptions import ClientDisconnected, ProtocolError

from .testing import Loop
from .client import ClientRunner, Fallback
from .server import new_connection, best_protocol_version
from .marshal import encoder, decoder

class Base(object):

    enc = b'Z'
    seq_type = list

    def setUp(self):
        super(Base, self).setUp()
        self.encode = encoder(self.enc)
        self.decode = decoder(self.enc)

    def unsized(self, data, unpickle=False):
        result = []
        while data:
            size, message = data[:2]
            data = data[2:]
            self.assertEqual(struct.unpack(">I", size)[0], len(message))
            if unpickle:
                message = self.decode(message)
            result.append(message)

        if len(result) == 1:
            result = result[0]
        return result

    def parse(self, data):
        return self.unsized(data, True)

    target = None
    def send(self, method, *args, **kw):
        target = kw.pop('target', self.target)
        called = kw.pop('called', True)
        no_output = kw.pop('no_output', True)
        self.assertFalse(kw)

        self.loop.protocol.data_received(
            sized(self.encode(0, True, method, args)))
        if target is not None:
            target = getattr(target, method)
            if called:
                target.assert_called_with(*args)
                target.reset_mock()
            else:
                self.assertFalse(target.called)
        if no_output:
            self.assertFalse(self.loop.transport.pop())

    def pop(self, count=None, parse=True):
        return self.unsized(self.loop.transport.pop(count), parse)

class ClientTests(Base, setupstack.TestCase, ClientRunner):

    maxDiff = None

    def tearDown(self):
        self.client.close()
        super(ClientTests, self)

    def start(self,
              addrs=(('127.0.0.1', 8200), ), loop_addrs=None,
              read_only=False,
              finish_start=False,
              ):
        # To create a client, we need to specify an address, a client
        # object and a cache.

        wrapper = mock.Mock()
        self.target = wrapper
        cache = MemoryCache()
        self.set_options(addrs, wrapper, cache, 'TEST', read_only, timeout=1)

        # We can also provide an event loop.  We'll use a testing loop
        # so we don't have to actually make any network connection.
        loop = Loop(addrs if loop_addrs is None else loop_addrs)
        self.setup_delegation(loop)
        self.assertFalse(wrapper.notify_disconnected.called)
        protocol = loop.protocol
        transport = loop.transport

        if finish_start:
            protocol.data_received(sized(self.enc + b'3101'))
            self.assertEqual(self.pop(2, False), self.enc + b'3101')
            self.respond(1, None)
            self.respond(2, 'a'*8)
            self.pop(4)
            self.assertEqual(self.pop(), (3, False, 'get_info', ()))
            self.respond(3, dict(length=42))

        return (wrapper, cache, self.loop, self.client, protocol, transport)

    def respond(self, message_id, result, async_=False):
        self.loop.protocol.data_received(
            sized(self.encode(message_id, async_, '.reply', result)))

    def wait_for_result(self, future, timeout):
        if future.done() and future.exception() is not None:
            raise future.exception()
        return future

    def testClientBasics(self):

        # Here, we'll go through the basic usage of the asyncio ZEO
        # network client.  The client is responsible for the core
        # functionality of a ZEO client storage.  The client storage
        # is largely just a wrapper around the asyncio client.

        wrapper, cache, loop, client, protocol, transport = self.start()
        self.assertFalse(wrapper.notify_disconnected.called)

        # The client isn't connected until the server sends it some data.
        self.assertFalse(client.connected.done() or transport.data)

        # The server sends the client it's protocol. In this case,
        # it's a very high one.  The client will send it's highest that
        # it can use.
        protocol.data_received(sized(self.enc + b'99999'))

        # The client sends back a handshake, and registers the
        # storage, and requests the last transaction.
        self.assertEqual(self.pop(2, False), self.enc + b'5')
        self.assertEqual(self.pop(), (1, False, 'register', ('TEST', False)))

        # The client isn't connected until it initializes it's cache:
        self.assertFalse(client.connected.done() or transport.data)

        # If we try to make calls while the client is *initially*
        # connecting, we get an error. This is because some dufus
        # decided to create a client storage without waiting for it to
        # connect.
        self.assertRaises(ClientDisconnected, self.call, 'foo', 1, 2)

        # When the client is reconnecting, it's ready flag is set to False and
        # it queues calls:
        client.ready = False
        f1 = self.call('foo', 1, 2)
        self.assertFalse(f1.done())

        # If we try to make an async call, we get an immediate error:
        self.assertRaises(ClientDisconnected, self.async_, 'bar', 3, 4)

        # The wrapper object (ClientStorage) hasn't been notified:
        self.assertFalse(wrapper.notify_connected.called)

        # Let's respond to the register call:
        self.respond(1, None)

        # The client requests the last transaction:
        self.assertEqual(self.pop(), (2, False, 'lastTransaction', ()))

        # We respond
        self.respond(2, 'a'*8)

        # After verification, the client requests info:
        self.assertEqual(self.pop(), (3, False, 'get_info', ()))
        self.respond(3, dict(length=42))

        # Now we're connected, the cache was initialized, and the
        # queued message has been sent:
        self.assertTrue(client.connected.done())
        self.assertEqual(cache.getLastTid(), 'a'*8)
        self.assertEqual(self.pop(), (4, False, 'foo', (1, 2)))

        # The wrapper object (ClientStorage) has been notified:
        wrapper.notify_connected.assert_called_with(client, {'length': 42})

        self.respond(4, 42)
        self.assertEqual(f1.result(), 42)

        # Now we can make async calls:
        f2 = self.async_('bar', 3, 4)
        self.assertTrue(f2.done() and f2.exception() is None)
        self.assertEqual(self.pop(), (0, True, 'bar', (3, 4)))

        # Loading objects gets special handling to leverage the cache.
        loaded = self.load_before(b'1'*8, maxtid)

        # The data wasn't in the cache, so we made a server call:
        self.assertEqual(self.pop(), ((b'1'*8, maxtid), False, 'loadBefore', (b'1'*8, maxtid)))
        # Note load_before uses the oid as the message id.
        self.respond((b'1'*8, maxtid), (b'data', b'a'*8, None))
        self.assertEqual(loaded.result(), (b'data', b'a'*8, None))

        # If we make another request, it will be satisfied from the cache:
        loaded = self.load_before(b'1'*8, maxtid)
        self.assertEqual(loaded.result(), (b'data', b'a'*8, None))
        self.assertFalse(transport.data)

        # Let's send an invalidation:
        self.send('invalidateTransaction', b'b'*8, self.seq_type([b'1'*8]))

        # Now, if we try to load current again, we'll make a server request.
        loaded = self.load_before(b'1'*8, maxtid)

        # Note that if we make another request for the same object,
        # the requests will be collapsed:
        loaded2 = self.load_before(b'1'*8, maxtid)

        self.assertEqual(self.pop(), ((b'1'*8, maxtid), False, 'loadBefore', (b'1'*8, maxtid)))
        self.respond((b'1'*8, maxtid), (b'data2', b'b'*8, None))
        self.assertEqual(loaded.result(), (b'data2', b'b'*8, None))
        self.assertEqual(loaded2.result(), (b'data2', b'b'*8, None))

        # Loading non-current data may also be satisfied from cache
        loaded = self.load_before(b'1'*8, b'b'*8)
        self.assertEqual(loaded.result(), (b'data', b'a'*8, b'b'*8))
        self.assertFalse(transport.data)
        loaded = self.load_before(b'1'*8, b'c'*8)
        self.assertEqual(loaded.result(), (b'data2', b'b'*8, None))
        self.assertFalse(transport.data)
        loaded = self.load_before(b'1'*8, b'_'*8)

        self.assertEqual(self.pop(), ((b'1'*8, b'_'*8), False, 'loadBefore', (b'1'*8, b'_'*8)))
        self.respond((b'1'*8, b'_'*8), (b'data0', b'^'*8, b'_'*8))
        self.assertEqual(loaded.result(), (b'data0', b'^'*8, b'_'*8))

        # When committing transactions, we need to update the cache
        # with committed data.  To do this, we pass a (oid, data, resolved)
        # iteratable to tpc_finish_threadsafe.

        tids = []
        def finished_cb(tid):
            tids.append(tid)

        committed = self.tpc_finish(
            b'd'*8,
            [(b'2'*8, 'committed 2', False),
             (b'1'*8, 'committed 3', True),
             (b'4'*8, 'committed 4', False),
             ],
            finished_cb)
        self.assertFalse(committed.done() or
                         cache.load(b'2'*8) or
                         cache.load(b'4'*8))
        self.assertEqual(cache.load(b'1'*8), (b'data2', b'b'*8))
        self.assertEqual(self.pop(),
                         (5, False, 'tpc_finish', (b'd'*8,)))
        self.respond(5, b'e'*8)
        self.assertEqual(committed.result(), b'e'*8)
        self.assertEqual(cache.load(b'1'*8), None)
        self.assertEqual(cache.load(b'2'*8), ('committed 2', b'e'*8))
        self.assertEqual(cache.load(b'4'*8), ('committed 4', b'e'*8))
        self.assertEqual(tids.pop(), b'e'*8)

        # If the protocol is disconnected, it will reconnect and will
        # resolve outstanding requests with exceptions:
        loaded = self.load_before(b'1'*8, maxtid)
        f1 = self.call('foo', 1, 2)
        self.assertFalse(loaded.done() or f1.done())
        self.assertEqual(
            self.pop(),
            [((b'11111111', b'\x7f\xff\xff\xff\xff\xff\xff\xff'),
              False, 'loadBefore', (b'1'*8, maxtid)),
             (6, False, 'foo', (1, 2))],
            )
        exc = TypeError(43)

        self.assertFalse(wrapper.notify_disconnected.called)
        wrapper.notify_connected.reset_mock()
        protocol.connection_lost(exc)
        wrapper.notify_disconnected.assert_called_with()

        self.assertTrue(isinstance(loaded.exception(), ClientDisconnected))
        self.assertEqual(loaded.exception().args, (exc,))
        self.assertTrue(isinstance(f1.exception(), ClientDisconnected))
        self.assertEqual(f1.exception().args, (exc,))

        # Because we reconnected, a new protocol and transport were created:
        self.assertTrue(protocol is not loop.protocol)
        self.assertTrue(transport is not loop.transport)
        protocol = loop.protocol
        transport = loop.transport

        # and we have a new incomplete connect future:
        self.assertFalse(client.connected.done() or transport.data)

        # This time we'll send a lower protocol version.  The client
        # will send it back, because it's lower than the client's
        # protocol:
        protocol.data_received(sized(self.enc + b'310'))
        self.assertEqual(self.unsized(transport.pop(2)), self.enc + b'310')
        self.assertEqual(self.pop(), (1, False, 'register', ('TEST', False)))
        self.assertFalse(wrapper.notify_connected.called)

        # If the register response is a tid, then the client won't
        # request lastTransaction
        self.respond(1, b'e'*8)
        self.assertEqual(self.pop(), (2, False, 'get_info', ()))
        self.respond(2, dict(length=42))

        # Because the server tid matches the cache tid, we're done connecting
        wrapper.notify_connected.assert_called_with(client, {'length': 42})
        self.assertTrue(client.connected.done() and not transport.data)
        self.assertEqual(cache.getLastTid(), b'e'*8)

        # Because we were able to update the cache, we didn't have to
        # invalidate the database cache:
        self.assertFalse(wrapper.invalidateTransaction.called)

        # The close method closes the connection and cache:
        client.close()
        self.assertTrue(transport.closed and cache.closed)

        # The client doesn't reconnect
        self.assertEqual(loop.protocol, protocol)
        self.assertEqual(loop.transport, transport)

    def test_cache_behind(self):
        wrapper, cache, loop, client, protocol, transport = self.start()

        cache.setLastTid(b'a'*8)
        cache.store(b'4'*8, b'a'*8, None, '4 data')
        cache.store(b'2'*8, b'a'*8, None, '2 data')

        self.assertFalse(client.connected.done() or transport.data)
        protocol.data_received(sized(self.enc + b'3101'))
        self.assertEqual(self.unsized(transport.pop(2)), self.enc + b'3101')
        self.respond(1, None)
        self.respond(2, b'e'*8)
        self.pop(4)

        # We have to verify the cache, so we're not done connecting:
        self.assertFalse(client.connected.done())
        self.assertEqual(self.pop(), (3, False, 'getInvalidations', (b'a'*8, )))
        self.respond(3, (b'e'*8, [b'4'*8]))

        self.assertEqual(self.pop(), (4, False, 'get_info', ()))
        self.respond(4, dict(length=42))

        # Now that verification is done, we're done connecting
        self.assertTrue(client.connected.done() and not transport.data)
        self.assertEqual(cache.getLastTid(), b'e'*8)

        # And the cache has been updated:
        self.assertEqual(cache.load(b'2'*8),
                         ('2 data', b'a'*8)) # unchanged
        self.assertEqual(cache.load(b'4'*8), None)

        # Because we were able to update the cache, we didn't have to
        # invalidate the database cache:
        self.assertFalse(wrapper.invalidateCache.called)

    def test_cache_way_behind(self):
        wrapper, cache, loop, client, protocol, transport = self.start()

        cache.setLastTid(b'a'*8)
        cache.store(b'4'*8, b'a'*8, None, '4 data')
        self.assertTrue(cache)

        self.assertFalse(client.connected.done() or transport.data)
        protocol.data_received(sized(self.enc + b'3101'))
        self.assertEqual(self.unsized(transport.pop(2)), self.enc + b'3101')
        self.respond(1, None)
        self.respond(2, b'e'*8)
        self.pop(4)

        # We have to verify the cache, so we're not done connecting:
        self.assertFalse(client.connected.done())
        self.assertEqual(self.pop(), (3, False, 'getInvalidations', (b'a'*8, )))

        # We respond None, indicating that we're too far out of date:
        self.respond(3, None)

        self.assertEqual(self.pop(), (4, False, 'get_info', ()))
        self.respond(4, dict(length=42))

        # Now that verification is done, we're done connecting
        self.assertTrue(client.connected.done() and not transport.data)
        self.assertEqual(cache.getLastTid(), b'e'*8)

        # But the cache is now empty and we invalidated the database cache
        self.assertFalse(cache)
        wrapper.invalidateCache.assert_called_with()

    def test_multiple_addresses(self):
        # We can pass multiple addresses to client constructor
        addrs = [('1.2.3.4', 8200), ('2.2.3.4', 8200)]
        wrapper, cache, loop, client, protocol, transport = self.start(
            addrs, ())

        # We haven't connected yet
        self.assertTrue(protocol is None and transport is None)

        # There are 2 connection attempts outstanding:
        self.assertEqual(sorted(loop.connecting), addrs)

        # We cause the first one to fail:
        loop.fail_connecting(addrs[0])
        self.assertEqual(sorted(loop.connecting), addrs[1:])

        # The failed connection is attempted in the future:
        delay, func, args, _ = loop.later.pop(0)
        self.assertTrue(1 <= delay <= 2)
        func(*args)
        self.assertEqual(sorted(loop.connecting), addrs)

        # Let's connect the second address
        loop.connect_connecting(addrs[1])
        self.assertEqual(sorted(loop.connecting), addrs[:1])
        protocol = loop.protocol
        transport = loop.transport
        protocol.data_received(sized(self.enc + b'3101'))
        self.assertEqual(self.unsized(transport.pop(2)), self.enc + b'3101')
        self.respond(1, None)

        # Now, when the first connection fails, it won't be retried,
        # because we're already connected.
        # (first in later is heartbeat)
        self.assertEqual(sorted(loop.later[1:]), [])
        loop.fail_connecting(addrs[0])
        self.assertEqual(sorted(loop.connecting), [])
        self.assertEqual(sorted(loop.later[1:]), [])

    def test_bad_server_tid(self):
        # If in verification we get a server_tid behind the cache's, make sure
        # we retry the connection later.
        wrapper, cache, loop, client, protocol, transport = self.start()
        cache.store(b'4'*8, b'a'*8, None, '4 data')
        cache.setLastTid('b'*8)
        protocol.data_received(sized(self.enc + b'3101'))
        self.assertEqual(self.unsized(transport.pop(2)), self.enc + b'3101')
        self.respond(1, None)
        self.respond(2, 'a'*8)
        self.pop()
        self.assertFalse(client.connected.done() or transport.data)
        delay, func, args, _ = loop.later.pop(1) # first in later is heartbeat
        self.assertTrue(8 < delay < 10)
        self.assertEqual(len(loop.later), 1) # first in later is heartbeat
        func(*args) # connect again
        self.assertFalse(protocol is loop.protocol)
        self.assertFalse(transport is loop.transport)
        protocol = loop.protocol
        transport = loop.transport
        protocol.data_received(sized(self.enc + b'3101'))
        self.assertEqual(self.unsized(transport.pop(2)), self.enc + b'3101')
        self.respond(1, None)
        self.respond(2, 'b'*8)
        self.pop(4)
        self.assertEqual(self.pop(), (3, False, 'get_info', ()))
        self.respond(3, dict(length=42))
        self.assertTrue(client.connected.done() and not transport.data)
        self.assertTrue(client.ready)

    def test_readonly_fallback(self):
        addrs = [('1.2.3.4', 8200), ('2.2.3.4', 8200)]
        wrapper, cache, loop, client, protocol, transport = self.start(
            addrs, (), read_only=Fallback)

        self.assertTrue(self.is_read_only())

        # We'll treat the first address as read-only and we'll let it connect:
        loop.connect_connecting(addrs[0])
        protocol, transport = loop.protocol, loop.transport
        protocol.data_received(sized(self.enc + b'3101'))
        self.assertEqual(self.unsized(transport.pop(2)), self.enc + b'3101')
        # We see that the client tried a writable connection:
        self.assertEqual(self.pop(),
                         (1, False, 'register', ('TEST', False)))
        # We respond with a read-only exception:
        self.respond(1, ('ZODB.POSException.ReadOnlyError', ()), True)
        self.assertTrue(self.is_read_only())

        # The client tries for a read-only connection:
        self.assertEqual(self.pop(), (2, False, 'register', ('TEST', True)))
        # We respond with successfully:
        self.respond(2, None)
        self.pop(2)
        self.respond(3, 'b'*8)
        self.assertTrue(self.is_read_only())

        # At this point, the client is ready and using the protocol,
        # and the protocol is read-only:
        self.assertTrue(client.ready)
        self.assertEqual(client.protocol, protocol)
        self.assertEqual(protocol.read_only, True)
        connected = client.connected

        # The client asks for info, and we respond:
        self.assertEqual(self.pop(), (4, False, 'get_info', ()))
        self.respond(4, dict(length=42))

        self.assertTrue(connected.done())

        # We connect the second address:
        loop.connect_connecting(addrs[1])
        loop.protocol.data_received(sized(self.enc + b'3101'))
        self.assertEqual(self.unsized(loop.transport.pop(2)), self.enc + b'3101')
        self.assertEqual(self.parse(loop.transport.pop()),
                         (1, False, 'register', ('TEST', False)))
        self.assertTrue(self.is_read_only())

        # We respond and the writable connection succeeds:
        self.respond(1, None)

        # at this point, a lastTransaction request is emitted:

        self.assertEqual(self.parse(loop.transport.pop()),
                         (2, False, 'lastTransaction', ()))
        self.assertFalse(self.is_read_only())

        # Now, the original protocol is closed, and the client is
        # no-longer ready:
        self.assertFalse(client.ready)
        self.assertFalse(client.protocol is protocol)
        self.assertEqual(client.protocol, loop.protocol)
        self.assertEqual(protocol.closed, True)
        self.assertTrue(client.connected is not connected)
        self.assertFalse(client.connected.done())
        protocol, transport = loop.protocol, loop.transport
        self.assertEqual(protocol.read_only, False)

        # Now, we finish verification
        self.respond(2, 'b'*8)
        self.respond(3, dict(length=42))
        self.assertTrue(client.ready)
        self.assertTrue(client.connected.done())

    def test_invalidations_while_verifying(self):
        # While we're verifying, invalidations are ignored
        wrapper, cache, loop, client, protocol, transport = self.start()
        protocol.data_received(sized(self.enc + b'3101'))
        self.assertEqual(self.unsized(transport.pop(2)), self.enc + b'3101')
        self.respond(1, None)
        self.pop(4)
        self.send('invalidateTransaction', b'b'*8, [b'1'*8], called=False)
        self.respond(2, b'a'*8)
        self.send('invalidateTransaction', b'c'*8, self.seq_type([b'1'*8]),
                  no_output=False)
        self.assertEqual(self.pop(), (3, False, 'get_info', ()))

        # We'll disconnect:
        protocol.connection_lost(Exception("lost"))
        self.assertTrue(protocol is not loop.protocol)
        self.assertTrue(transport is not loop.transport)
        protocol = loop.protocol
        transport = loop.transport

        # Similarly, invalidations aren't processed while reconnecting:

        protocol.data_received(sized(self.enc + b'3101'))
        self.assertEqual(self.unsized(transport.pop(2)), self.enc + b'3101')
        self.respond(1, None)
        self.pop(4)
        self.send('invalidateTransaction', b'd'*8, [b'1'*8], called=False)
        self.respond(2, b'c'*8)
        self.send('invalidateTransaction', b'e'*8, self.seq_type([b'1'*8]),
                  no_output=False)
        self.assertEqual(self.pop(), (3, False, 'get_info', ()))

    def test_flow_control(self):
        # When sending a lot of data (blobs), we don't want to fill up
        # memory behind a slow socket. Asycio's flow control helper
        # seems a bit complicated. We'd rather pass an iterator that's
        # consumed as we can.

        wrapper, cache, loop, client, protocol, transport = self.start(
            finish_start=True)

        # Give the transport a small capacity:
        transport.capacity = 2
        self.async_('foo')
        self.async_('bar')
        self.async_('baz')
        self.async_('splat')

        # The first 2 were sent, but the remaining were queued.
        self.assertEqual(self.pop(),
                         [(0, True, 'foo', ()), (0, True, 'bar', ())])

        # But popping them allowed sending to resume:
        self.assertEqual(self.pop(),
                         [(0, True, 'baz', ()), (0, True, 'splat', ())])

        # This is especially handy with iterators:
        self.async_iter((name, ()) for name in 'abcde')
        self.assertEqual(self.pop(), [(0, True, 'a', ()), (0, True, 'b', ())])
        self.assertEqual(self.pop(), [(0, True, 'c', ()), (0, True, 'd', ())])
        self.assertEqual(self.pop(), (0, True, 'e', ()))
        self.assertEqual(self.pop(), [])

    def test_bad_protocol(self):
        wrapper, cache, loop, client, protocol, transport = self.start()
        with mock.patch("ZEO.asyncio.client.logger.error") as error:
            self.assertFalse(error.called)
            protocol.data_received(sized(self.enc + b'200'))
            self.assertTrue(isinstance(error.call_args[0][1], ProtocolError))

    def test_get_peername(self):
        wrapper, cache, loop, client, protocol, transport = self.start(
            finish_start=True)
        self.assertEqual(client.get_peername(), '1.2.3.4')

    def test_call_async_from_same_thread(self):
        # There are a few (1?) cases where we call into client storage
        # where it needs to call back asyncronously. Because we're
        # calling from the same thread, we don't need to use a futurte.
        wrapper, cache, loop, client, protocol, transport = self.start(
            finish_start=True)

        client.call_async_from_same_thread('foo', 1)
        self.assertEqual(self.pop(), (0, True, 'foo', (1, )))

    def test_ClientDisconnected_on_call_timeout(self):
        wrapper, cache, loop, client, protocol, transport = self.start()
        self.wait_for_result = super(ClientTests, self).wait_for_result
        self.assertRaises(ClientDisconnected, self.call, 'foo')
        client.ready = False
        self.assertRaises(ClientDisconnected, self.call, 'foo')

    def test_errors_in_data_received(self):
        # There was a bug in ZEO.async.client.Protocol.data_recieved
        # that caused it to fail badly if errors were raised while
        # handling data.

        wrapper, cache, loop, client, protocol, transport =self.start(
            finish_start=True)

        wrapper.receiveBlobStart.side_effect = ValueError('test')

        chunk = 'x' * 99999
        try:
            loop.protocol.data_received(
                sized(
                    self.encode(0, True, 'receiveBlobStart', ('oid', 'serial'))
                    ) +
                sized(
                    self.encode(
                        0, True, 'receiveBlobChunk', ('oid', 'serial', chunk))
                    )
                )
        except ValueError:
            pass
        loop.protocol.data_received(sized(
            self.encode(0, True, 'receiveBlobStop', ('oid', 'serial'))
            ))
        wrapper.receiveBlobChunk.assert_called_with('oid', 'serial', chunk)
        wrapper.receiveBlobStop.assert_called_with('oid', 'serial')

    def test_heartbeat(self):
        # Protocols run heartbeats on a configurable (sort of)
        # heartbeat interval, which defaults to every 60 seconds.
        wrapper, cache, loop, client, protocol, transport = self.start(
            finish_start=True)

        delay, func, args, handle = loop.later.pop()
        self.assertEqual(
            (delay, func, args, handle),
            (60, protocol.heartbeat, (), protocol.heartbeat_handle),
            )
        self.assertFalse(loop.later or handle.cancelled)

        # The heartbeat function sends heartbeat data and reschedules itself.
        func()
        self.assertEqual(self.pop(), (-1, 0, '.reply', None))
        self.assertTrue(protocol.heartbeat_handle != handle)

        delay, func, args, handle = loop.later.pop()
        self.assertEqual(
            (delay, func, args, handle),
            (60, protocol.heartbeat, (), protocol.heartbeat_handle),
            )
        self.assertFalse(loop.later or handle.cancelled)

        # The heartbeat is cancelled when the protocol connection is lost:
        protocol.connection_lost(None)
        self.assertTrue(handle.cancelled)

class MsgpackClientTests(ClientTests):
    enc = b'M'
    seq_type = tuple

class MemoryCache(object):

    def __init__(self):
        # { oid -> [(start, end, data)] }
        self.data = collections.defaultdict(list)
        self.last_tid = None
        self._lock = RLock()

    clear = __init__

    closed = False
    def close(self):
        self.closed = True

    def __len__(self):
        return len(self.data)

    def load(self, oid):
        revisions = self.data[oid]
        if revisions:
            start, end, data = revisions[-1]
            if not end:
                return data, start
        return None

    def store(self, oid, start_tid, end_tid, data):
        assert start_tid is not None
        revisions = self.data[oid]
        data = (start_tid, end_tid, data)
        if not revisions or data != revisions[-1]:
            revisions.append(data)
        revisions.sort()

    def loadBefore(self, oid, tid):
        for start, end, data in self.data[oid]:
            if start < tid and (end is None or end >= tid):
                return data, start, end

    def invalidate(self, oid, tid):
        revisions = self.data[oid]
        if revisions:
            if tid is None:
                del revisions[:]
            else:
                start, end, data = revisions[-1]
                if end is None:
                    revisions[-1] = start, tid, data

    def getLastTid(self):
        return self.last_tid

    def setLastTid(self, tid):
        self.last_tid = tid

class ServerTests(Base, setupstack.TestCase):

    # The server side of things is pretty simple compared to the
    # client, because it's the clien't job to make and keep
    # connections. Servers are pretty passive.

    def connect(self, finish=False):
        protocol = server_protocol(self.enc == b'M')
        self.loop = protocol.loop
        self.target = protocol.zeo_storage
        if finish:
            self.assertEqual(self.pop(parse=False),
                             self.enc + best_protocol_version)
            protocol.data_received(sized(self.enc + b'5'))
        return protocol

    message_id = 0
    target = None
    def call(self, meth, *args, **kw):
        if kw:
            expect = kw.pop('expect', self)
            target = kw.pop('target', self.target)
            self.assertFalse(kw)

            if target is not None:
                target = getattr(target, meth)
                if expect is not self:
                    target.return_value = expect

        self.message_id += 1
        self.loop.protocol.data_received(
            sized(self.encode(self.message_id, False, meth, args)))

        if target is not None:
            target.assert_called_once_with(*args)
            target.reset_mock()

        if expect is not self:
            self.assertEqual(self.pop(),
                             (self.message_id, False, '.reply', expect))

    def testServerBasics(self):
        # A simple listening thread accepts connections.  It creats
        # asyncio connections by calling ZEO.asyncio.new_connection:
        protocol = self.connect()
        self.assertFalse(protocol.zeo_storage.notify_connected.called)

        # The server sends it's protocol.
        self.assertEqual(self.pop(parse=False),
                         self.enc + best_protocol_version)

        # The client sends it's protocol:
        protocol.data_received(sized(self.enc + b'5'))

        self.assertEqual(protocol.protocol_version, self.enc + b'5')

        protocol.zeo_storage.notify_connected.assert_called_once_with(protocol)

        # The client registers:
        self.call('register', False, expect=None)

        # It does other things, like, send hearbeats:
        protocol.data_received(sized(b'(J\xff\xff\xff\xffK\x00U\x06.replyNt.'))

        # The client can make async calls:
        self.send('register')

        # Let's close the connection
        self.assertFalse(protocol.zeo_storage.notify_disconnected.called)
        protocol.connection_lost(None)
        protocol.zeo_storage.notify_disconnected.assert_called_once_with()

    def test_invalid_methods(self):
        protocol = self.connect(True)
        protocol.zeo_storage.notify_connected.assert_called_once_with(protocol)

        # If we try to call a methid that isn't in the protocol's
        # white list, it will disconnect:
        self.assertFalse(protocol.loop.transport.closed)
        self.call('foo', target=None)
        self.assertTrue(protocol.loop.transport.closed)

class MsgpackServerTests(ServerTests):
    enc = b'M'
    seq_type = tuple

def server_protocol(msgpack,
                    zeo_storage=None,
                    protocol_version=None,
                    addr=('1.2.3.4', '42'),
                    ):
    if zeo_storage is None:
        zeo_storage = mock.Mock()
    loop = Loop()
    sock = () # anything not None
    new_connection(loop, addr, sock, zeo_storage, msgpack)
    if protocol_version:
        loop.protocol.data_received(sized(protocol_version))
    return loop.protocol

def response(*data):
    return sized(self.encode(*data))

def sized(message):
    return struct.pack(">I", len(message)) + message

class Logging(object):

    def __init__(self, level=logging.ERROR):
        self.level = level

    def __enter__(self):
        self.handler = logging.StreamHandler()
        logging.getLogger().addHandler(self.handler)
        logging.getLogger().setLevel(self.level)

    def __exit__(self, *args):
        logging.getLogger().removeHandler(self.handler)
        logging.getLogger().setLevel(logging.NOTSET)

def test_suite():
    suite = unittest.TestSuite()
    suite.addTest(unittest.makeSuite(ClientTests))
    suite.addTest(unittest.makeSuite(ServerTests))
    suite.addTest(unittest.makeSuite(MsgpackClientTests))
    suite.addTest(unittest.makeSuite(MsgpackServerTests))
    return suite


import os
import re
import sys
import json
from termcolor import colored
from string import punctuation
import argparse
from ntpath import split as ntsplit
from nltk.corpus.reader import PlaintextCorpusReader
from nltk.stem.snowball import FrenchStemmer, SnowballStemmer

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.metrics.pairwise import cosine_similarity

import operator

# def getVectorSpace(cleanSet):
#     vocab = {}
#     for data in cleanSet:
#         for word in data.split():
#             print("woard", word)
#             vocab[data] = 0
#     return vocab.keys()

def calculateSimilarity(sentence, doc):
    if doc == []:
        return 0
    vocab = {}
    for word in sentence.split(" "):
        vocab[word] = 0

    docInOneSentence = ""
    for t in doc:
        docInOneSentence += t + " "
        for word in t.split():
            vocab[word] = 0

    cv = CountVectorizer(vocabulary=vocab.keys())

    docVector = cv.fit_transform([docInOneSentence])
    sentenceVector = cv.fit_transform([sentence])
    return cosine_similarity(docVector, sentenceVector)[0][0]

def load_stopwords(stopwords_file, lang):
    with open(stopwords_file) as f:
        return json.load(f)[lang]

def cleanData(sentence):
    # remove stopwords
    # sentence = re.sub('[^A-Za-z0-9 ]+', '', sentence)
    # sentence filter(None, re.split("[.!?", setence))
    ret = []
    sentence = stemmer.stem(sentence)
    for word in sentence.split():
        if not word in stopwords:
            ret.append(word)
    return " ".join(ret)

def stemmize(stemmer, sent, stopwords):
    clean_sent = []
    for word in sent:
        stem = stemmer.stem(word)
        if word not in stopwords and word not in punctuation:
            clean_sent.append(stem)
    return clean_sent

def load_sentences(text_file, stopwords, lang):
    path, f = ntsplit(text_file)
    reader = PlaintextCorpusReader(path, f)
    sentences = [sent for sent in reader.sents()]
    clean = []
    originalSentenceOf = {}
    if lang == "fr":
        stemmer = FrenchStemmer()
    elif lang == "en":
        stemmer = SnowballStemmer("english")
    # Data cleansing
    for sent in sentences:
        s = stemmize(stemmer, sent, stopwords)
        clean.append(" ".join(s))
        originalSentenceOf[clean[-1]] = sent
    setClean = set(clean)
    return setClean, originalSentenceOf, sentences, clean

def load_data(stopwords_file, text_file, lang):
    stopwords = load_stopwords(stopwords_file, lang)
    setClean, originalSentenceOf, sentences, clean = load_sentences(
        text_file, stopwords, lang
    )

    return stopwords, setClean, originalSentenceOf, sentences, clean

def compute_similarity_scores(clean, setClean):
    # calculate Similarity score each sentence with whole documents
    scores = {}
    for data in clean:
        temp_doc = setClean - set([data])
        score = calculateSimilarity(data, list(temp_doc))
        scores[data] = score
    return scores

def summarize_mmr(scores, sentences, percent):
    # calculate MMR
    n = percent * len(sentences) / 100
    alpha = 0.5
    summarySet = []
    while n > 0:
        mmr = {}
        for sentence in scores.keys():
            if not sentence in summarySet:
                mmr[sentence] = alpha * scores[sentence] - (
                    1 - alpha
                ) * calculateSimilarity(sentence, summarySet)
        selected = max(mmr.items(), key=operator.itemgetter(1))[0]
        summarySet.append(selected)
        n -= 1
    return summarySet

def show_results(summarySet, originalSentenceOf, clean):

    print("\nSummary:\n")
    for sentence in summarySet:
        print(" ".join(originalSentenceOf[sentence]))
    print()

    print("=============================================================")
    print("\nOriginal Passages:\n")
    for sentence in clean:
        if sentence in summarySet:
            print(colored(" ".join(originalSentenceOf[sentence]), "red"))
        else:
            print(" ".join(originalSentenceOf[sentence]))

def run_mmr(stopwords_file, text_file, lang, percent):
    stopwords, setClean, originalSentenceOf, sentences, clean = load_data(
        stopwords_file, text_file, lang
    )
    scores = compute_similarity_scores(clean, setClean)
    summary = summarize_mmr(scores, sentences, percent)
    show_results(summary, originalSentenceOf, clean)

def main():
    def parse_arguments():
        parser = argparse.ArgumentParser(description="simple mmr for French")
        parser.add_argument(
            "-t", "--text", help="file to summarize", default="sample.txt"
        )
        parser.add_argument(
            "-s",
            "--stopwords",
            help="json file containing stopwords for multiple languages ",
            default="stopwords.json",
        )
        parser.add_argument(
            "-p",
            "--percent",
            type=int,
            help="percentage of sentences to use for summary",
            default=20,
        )
        parser.add_argument(
            "-l", "--lang", type=str, help="language of the input text", default="fr"
        )

        args = parser.parse_args()

        if args.lang.lower() in ("french", "fr"):
            args.lang = "fr"
        elif args.lang.lower() in ("english", "en"):
            args.lang = "en"
        else:
            raise Exception("Unknown lang ", args.lang)

        return args

    args = parse_arguments()
    stopwords_file = args.stopwords
    text_file = args.text
    percent = args.percent

    lang = args.lang
    print("text file :", text_file, file=sys.stderr)
    print("stopwords file", stopwords_file, file=sys.stderr, end="\n\n\n")

    run_mmr(stopwords_file, text_file, lang)

if __name__ == "__main__":
    main()


# coding=utf-8
# 对全行业的原始数据进行聚类/清洗全零行
# 将聚类结果中位数等结果，进行可视化显示/散点图/横向条形图/折线图

import pandas as pd
import numpy as np
from pandas import DataFrame
from sklearn.cluster import KMeans
import matplotlib.pyplot as plt

A = pd.read_excel(r'D:\Program Files\JetBrains\PyCharmFile\Cluster\Data\d2.xlsx', sheet_name="Sheet1", header=0)
tabtop = A.columns.values.tolist()        # 读出表头
B = np.array(A)                           # 转化为矩阵
C1 = B[0:1543, 0:1]                       # 取出第一列数据
C2 = C1.reshape(-1)                       # 转为行矩阵
C = C2.astype(str)                        # 变换为字符串形式
lenth1 = len(C)
count = []
temp = []
linelabel = list(range(1, 13))
year1 = ['2016年', '2017年', '2018年']

print(C)

for k in range(lenth1-1):
    if (C[k])[0:2] == (C[k+1])[0:2]:
        temp.append((C[k])[0:2])
        continue
    else:
        count.append(k+1)
count.append(lenth1+1)
print(count)              # 取出第一列数据的聚类起始位置
temp = sorted(set(temp), key=temp.index)     # 去掉列表中的重复值
print(temp)

t = 0
b = 0
for i in count:
    a = b
    b = i
    x = 6
    y = 18
    for j in year1:
        # 对某一行业的进行聚类(三次)
        E1 = B[a:b, x:y]
        E2 = DataFrame(E1)
        D1 = E2.loc[~(E2 == 0).all(axis=1), :]
        E3 = np.array(D1)
        model = KMeans(n_clusters=4)           # 聚类函数参数设置
        model.fit(D1)                          # 开始聚类
        R = DataFrame(model.cluster_centers_)  # 找出聚类中心
        D = DataFrame(D1)
        label1 = model.labels_
        label2 = list(set(label1))     # 删除矩阵中重复值
        len5 = len(label2)             # 类别数的长度
        len4 = D.shape[0]

        m1 = np.median(R, axis=0)    # 求出每列的中位数
        Rm1 = np.vstack((R, m1))     # 将中位数的列合并
        len1 = Rm1.shape[0]          # 求出行的数目
        print(len1)

        m2 = np.median(D, axis=0)   # 求出每列的中位数
        Dm2 = np.vstack((D, m2))    # 将中位数的列合并
        len2 = Dm2.shape[0]         # 求出行的数目
        print(len2)

        Dm21 = np.vstack((Dm2, m1))  # 将原始数据和两组中位数合并
        len3 = Dm21.shape[0]  # 求出行的数目

        print(len3)

        plt.rcParams['font.sans-serif'] = ['SimHei']  # 显示正常中文标签
        plt.rcParams['axes.unicode_minus'] = False  # 用来正常显示负号
        color1 = ['red', 'blue', 'orange', 'green', 'black', 'gray', 'pink', 'brown',  'yellow', 'purple']
        plt.figure(figsize=(10, 6))
        ind = np.arange(len(linelabel))  # the x locations for the groups
        width = 0.2                      # the width of the bars
        dis = [-0.3, -0.15, 0, 0.15, 0.3]
        for i1 in range(len1):
            if i1 == (len1 - 1):
                plt.barh(ind + dis[i1], Rm1[i1], width, color=color1[i1], label='聚类中位数')
            else:
                plt.barh(ind + dis[i1], Rm1[i1], width, color=color1[i1], label='类别' + str(i1 + 1))

        plt.legend(loc='best')
        plt.yticks(ind, ('1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12'))  # x轴的刻度
        title1 = j + '企业月用电量聚类中心图'
        plt.title(title1)
        plt.xlabel('聚类中心值')
        plt.ylabel('月份')

        name1 = temp[t] + '行业' + j + '企业聚类1'
        plt.savefig(name1)
        plt.close()
        plt.figure()
        for i2 in range(len2):
            if i2 == (len2 - 1):
                plt.plot(linelabel, Dm2[i2], color='blue', linestyle='--', marker='^', linewidth=0.5, label='用电量中位数')
            elif i2 == (len2 - 2):
                lengedname = '各企业'
                plt.plot(linelabel, Dm2[i2], color='darkgray', linewidth=1.0, label=lengedname)
            else:
                plt.plot(linelabel, Dm2[i2], color='darkgray', linewidth=1.0)

        plt.legend(loc='best')
        title2 = j + '企业月用电量图1'
        plt.title(title2)
        plt.ylabel('用电量/千瓦时')
        plt.xlabel('月份')
        name2 = temp[t] + '行业' + j + '企业聚类2'
        plt.savefig(name2)
        plt.close()

        plt.figure()
        for i3 in range(len3):
            if i3 == (len3 - 1):
                plt.plot(linelabel, Dm21[i3], color='red', linestyle='--', marker='*', linewidth=0.5, label='聚类中位数')
            elif i3 == (len3 - 2):
                plt.plot(linelabel, Dm21[i3], color='blue', linestyle='--', marker='^', linewidth=0.5, label='用电量中位数')
            elif i3 == (len3 - 3):
                lengedname = '各企业'
                plt.plot(linelabel, Dm21[i3], color='darkgray', linewidth=1.0, label=lengedname)
            else:
                plt.plot(linelabel, Dm21[i3], color='darkgray', linewidth=1.0)

        plt.legend(loc='best')
        title3 = j + '企业月用电量图'
        plt.title(title3)
        plt.ylabel('用电量/千瓦时')
        plt.xlabel('月份')
        name3 = temp[t] + '行业' + j + '企业聚类3'
        plt.savefig(name3)
        plt.close()

        plt.figure()
        for i4 in range(len5):
            count = 0
            for j2 in range(len4):
                if (label1[j2] == label2[i4]) and (count == 0):
                    plt.scatter(linelabel, E3[j2], color=color1[i4], linewidth=1.0, label='类别' + str(i4 + 1))
                    count = count + 1
                if (label1[j2] == label2[i4]) and (count > 0):
                    plt.scatter(linelabel, E3[j2], color=color1[i4], linewidth=1.0)

        plt.legend(loc='best')
        title4 = j + '企业月用电量聚类类别图'
        plt.title(title4)
        plt.ylabel('用电量/千瓦时')
        plt.xlabel('月份')
        name4 = temp[t] + '行业' + j + '企业聚类4'
        plt.savefig(name4)
        plt.close()
        x = y
        y = y + 12
    t = t + 1


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# checks if diffusion coefficients from umbrellas are similar to standard trajectories

protname = ['1E0G','1R69','SH3']
std_path_1E0G = '/home/tab10/work/simdata/6-20-15_native/'+protname[0]+'ca_avg_gaussian/'+protname[0]+'/iteration_0/'
umb_path_1E0G = '/home/tab10/work/simdata/6-20-15_native/'+protname[0]+'ca_avg_gaussian/'+protname[0]+'/umbrellas/'
std_path_1R69 = '/home/tab10/work/simdata/6-20-15_native/'+protname[1]+'ca_avg_gaussian/'+protname[1]+'/iteration_0/'
umb_path_1R69 = '/home/tab10/work/simdata/6-20-15_native/'+protname[1]+'ca_avg_gaussian/'+protname[1]+'/umbrellas/'
std_path_SH3 = '/home/tab10/work/simdata/6-20-15_native/'+protname[2]+'ca_avg_gaussian/'+protname[2]+'/iteration_0/'
umb_path_SH3 = '/home/tab10/work/simdata/6-20-15_native/'+protname[2]+'ca_avg_gaussian/'+protname[2]+'/umbrellas/'
model = ['Standard Trajectory', 'Umbrella Trajectory w/ integration', 'Umbrella Trajectory w/ fitting']

std_paths = [std_path_1E0G,std_path_1R69,std_path_SH3]
umb_paths = [umb_path_1E0G,umb_path_1R69,umb_path_SH3]

import os
import numpy as np
import matplotlib.pyplot as plt

#start = 120.45 # start and stop temps
#stop = 124.45
#temps = np.arange(start,stop+1,1)

std_data_1E0G = np.zeros((5,19))
std_data_1R69 = np.zeros((5,19))
std_data_SH3 = np.zeros((5,19))
umb_data_1E0G = np.zeros((5,3))
umb_data_1R69 = np.zeros((5,3))
umb_data_SH3 = np.zeros((5,3))
umb_data_1E0G_fits = np.zeros((5,3))
umb_data_1R69_fits = np.zeros((5,3))
umb_data_SH3_fits = np.zeros((5,3))
temps_1E0G = []
temps_1R69 = []
temps_SH3 = []
################################
os.chdir(std_path_1E0G)
with open('long_temps') as f:
    temps_1E0G = f.readlines()
for i in range(len(temps_1E0G)):
    temps_1E0G[i] = temps_1E0G[i].split('_')[0]
temps_1E0G = np.sort(list(set(temps_1E0G)))
std_data_1E0G[:,:] = np.loadtxt('TPTanalysis',skiprows=1)
fold_1E0G = np.loadtxt('short_Tf')
x_1E0G = std_data_1E0G[:,0]/fold_1E0G # T/T_f

os.chdir(std_path_1R69)
with open('long_temps') as f:
    temps_1R69 = f.readlines()
for i in range(len(temps_1R69)):
    temps_1R69[i] = temps_1R69[i].split('_')[0]
temps_1R69 = np.sort(list(set(temps_1R69)))
std_data_1R69[:,:] = np.loadtxt('TPTanalysis',skiprows=1)
fold_1R69 = np.loadtxt('short_Tf')
x_1R69 = std_data_1R69[:,0]/fold_1R69 # T/T_f

os.chdir(std_path_SH3)
with open('long_temps') as f:
    temps_SH3 = f.readlines()
for i in range(len(temps_SH3)):
    temps_SH3[i] = temps_SH3[i].split('_')[0]
temps_SH3 = np.sort(list(set(temps_SH3)))
std_data_SH3[:,:] = np.loadtxt('TPTanalysis',skiprows=1)
fold_SH3 = np.loadtxt('short_Tf')
x_SH3 = std_data_SH3[:,0]/fold_SH3 # T/T_f
#################################
os.chdir(umb_path_1E0G)
for i in range(len(temps_1E0G)):
    for j in [0,1,2]:
        os.chdir('%s_%s' % (temps_1E0G[i],j+1))
        umb_data_1E0G[i,j] = np.loadtxt('diff_umb_D')[2] # assumes TS umbrella is position 2
        umb_data_1E0G_fits[i,j] = np.loadtxt('diff_umb_D_fit')[2] # assumes TS umbrella is position 2
        os.chdir('..')

umb_data_1E0G_mean = np.mean(umb_data_1E0G,axis=1)
umb_data_1E0G_std = np.std(umb_data_1E0G,axis=1)
umb_data_1E0G_fits_mean = np.mean(umb_data_1E0G_fits,axis=1)
umb_data_1E0G_fits_std = np.std(umb_data_1E0G_fits,axis=1)

os.chdir(umb_path_1R69)
for i in range(len(temps_1R69)):
    for j in [0,1,2]:
        os.chdir('%s_%s' % (temps_1R69[i],j+1))
        umb_data_1R69[i,j] = np.loadtxt('diff_umb_D')[2] # assumes TS umbrella is position 2
        umb_data_1R69_fits[i,j] = np.loadtxt('diff_umb_D_fit')[2] # assumes TS umbrella is position 2
        os.chdir('..')

umb_data_1R69_mean = np.mean(umb_data_1R69,axis=1)
umb_data_1R69_std = np.std(umb_data_1R69,axis=1)
umb_data_1R69_fits_mean = np.mean(umb_data_1R69_fits,axis=1)
umb_data_1R69_fits_std = np.std(umb_data_1R69_fits,axis=1)

os.chdir(umb_path_SH3)
for i in range(len(temps_SH3)):
    for j in [0,1,2]:
        os.chdir('%s_%s' % (temps_SH3[i],j+1))
        umb_data_SH3[i,j] = np.loadtxt('diff_umb_D')[2] # assumes TS umbrella is position 2
        umb_data_SH3_fits[i,j] = np.loadtxt('diff_umb_D_fit')[2] # assumes TS umbrella is position 2
        os.chdir('..')

umb_data_SH3_mean = np.mean(umb_data_SH3,axis=1)
umb_data_SH3_std = np.std(umb_data_SH3,axis=1)
umb_data_SH3_fits_mean = np.mean(umb_data_SH3_fits,axis=1)
umb_data_SH3_fits_std = np.std(umb_data_SH3_fits,axis=1)
##################################
os.chdir('/home/tab10/scratch/')
if not os.path.exists('plots'):
    os.makedirs('plots')
os.chdir('plots')

f, (ax1, ax2, ax3) = plt.subplots(1, 3)
ax1.errorbar(x_1E0G,std_data_1E0G[:,3],yerr=std_data_1E0G[:,4],label=protname[0]+' '+model[0],lw=1.5)
ax2.errorbar(x_1E0G,umb_data_1E0G_mean,yerr=umb_data_1E0G_std,label=protname[0]+' '+model[1],lw=1.5)
ax1.errorbar(x_1R69,std_data_1R69[:,3],yerr=std_data_1R69[:,4],label=protname[1]+' '+model[0],lw=1.5)
ax2.errorbar(x_1R69,umb_data_1R69_mean,yerr=umb_data_1R69_std,label=protname[1]+' '+model[1],lw=1.5)
ax1.errorbar(x_SH3,std_data_SH3[:,3],yerr=std_data_SH3[:,4],label=protname[2]+' '+model[0],lw=1.5)
ax2.errorbar(x_SH3[:-1],umb_data_SH3_mean[:-1],yerr=umb_data_SH3_std[:-1],label=protname[2]+' '+model[1],lw=1.5)
ax3.errorbar(x_1E0G,umb_data_1E0G_fits_mean,yerr=umb_data_1E0G_fits_std,label=protname[0]+' '+model[2],lw=1.5)
ax3.errorbar(x_1R69,umb_data_1R69_fits_mean,yerr=umb_data_1R69_fits_std,label=protname[1]+' '+model[2],lw=1.5)
ax3.errorbar(x_SH3[:-1],umb_data_SH3_fits_mean[:-1],yerr=umb_data_SH3_fits_std[:-1],label=protname[2]+' '+model[2],lw=1.5)
ax1.set_title('D* from $t_{TP}$')
ax2.set_title('D* from umbrellas (integration)')
ax3.set_title('D* from umbrellas (fitting)')
ax1.set_xlabel('$T/T_f$')
ax2.set_xlabel('$T/T_f$')
ax3.set_xlabel('$T/T_f$')
ax1.set_ylabel('$D^*$ ($Q^{2}frames^{-1}$)')
plt.legend(prop={'size':10})
plt.savefig('diff_compare.png')
plt.savefig('diff_compare.pdf')
plt.show()


__copyright__ = "Copyright (c) 2020 Jina AI Limited. All rights reserved."
__license__ = "Apache-2.0"

import ctypes
import random
from typing import Iterator, Union

from ...helper import batch_iterator
from ...proto import jina_pb2

def _generate(data: Union[Iterator[bytes], Iterator['jina_pb2.Document']], batch_size: int = 0,
              first_doc_id: int = 0, first_request_id: int = 0,
              random_doc_id: bool = False, mode: str = 'index', top_k: int = 50,
              in_proto: bool = False,
              *args, **kwargs) -> Iterator['jina_pb2.Message']:
    for pi in batch_iterator(data, batch_size):
        req = jina_pb2.Request()
        req.request_id = first_request_id

        if mode == 'search':
            if top_k <= 0:
                raise ValueError('"top_k: %d" is not a valid number' % top_k)
            else:
                req.search.top_k = top_k

        for raw_bytes in pi:
            d = getattr(req, mode).docs.add()
            if in_proto:
                d.CopyFrom(raw_bytes)
            else:
                d.raw_bytes = raw_bytes
            d.doc_id = first_doc_id if not random_doc_id else random.randint(0, ctypes.c_uint(-1).value)
            d.weight = 1.0
            first_doc_id += 1
        yield req
        first_request_id += 1

def index(*args, **kwargs):
    """Generate indexing request"""
    yield from _generate(*args, **kwargs)

def train(*args, **kwargs):
    """Generate training request """
    yield from _generate(*args, **kwargs)
    req = jina_pb2.Request()
    req.request_id = 1
    req.train.flush = True
    yield req

def search(*args, **kwargs):
    """Generate search request """
    yield from _generate(*args, **kwargs)


import sqlite3

class Sqlite3ORM:
    fields = []
    dbpath = ""
    dbtable = ""
    create = ""

    def __init__(self):
        raise NotImplementedError

    def save(self):
        if self.pk is None:
            self._insert()
        else:
            self._update()

    @classmethod
    def _create_insert(cls):
        columnlist = ", ".join(cls.fields)
        qmarks = ", ".join("?" for val in cls.fields)
        SQL = """ INSERT INTO {tablename} ({columnlist})
        VALUES ({qmarks}) """
        return SQL.format(tablename=cls.dbtable, columnlist=columnlist, qmarks=qmarks)

    def _insert(self):
        with sqlite3.connect(self.dbpath) as conn:
            cur = conn.cursor()
            SQL = self._create_insert()
            propvals = [getattr(self, propname) for propname in self.fields]

            cur.execute(SQL, propvals)
            self.pk = cur.lastrowid

    @classmethod
    def _create_update(cls):
        """ TODO: IMPLEMENT THIS. Return a generic UPDATE SQL command like
        _create_insert did. You will want to be updating WHERE pk = ? """
        # field=?
        update_column_list = ", ".join(field+"=?" for field in cls.fields)
        SQL = """
UPDATE {tablename} SET {update_column_list} WHERE pk = ?; """
        return SQL.format(tablename=cls.dbtable, update_column_list=update_column_list)

    def _update(self):
        with sqlite3.connect(self.dbpath) as conn:
            cur = conn.cursor()
            SQL = self._create_update()
            propvals = [getattr(self, field) for field in self.fields + ["pk"]]
            cur.execute(SQL, propvals)

    @classmethod
    def one_where(cls, whereclause="TRUE", values=tuple()):
        SQL = "SELECT * FROM {tablename} WHERE " + whereclause
        with sqlite3.connect(cls.dbpath) as conn:
            conn.row_factory = sqlite3.Row
            cur = conn.cursor()

            cur.execute(SQL.format(tablename=cls.dbtable), values)
            row = cur.fetchone()
            if row is None:
                return None
            return cls(**row)

    @classmethod
    def many_where(cls, whereclause="TRUE", values=tuple()):
        """ equivalent of one_where but with fetchall, returns a list of objects or an
        empty list """
        SQL = "SELECT * FROM {tablename} WHERE " + whereclause
        with sqlite3.connect(cls.dbpath) as conn:
            conn.row_factory = sqlite3.Row
            cur = conn.cursor()

            cur.execute(SQL.format(tablename=cls.dbtable), values)
            rows = cur.fetchall()
            return [cls(**row) for row in rows]

    @classmethod
    def from_pk(cls, pk):
        return cls.one_where("pk=?", (pk,))

    @classmethod
    def all(cls):
        """ return a list of every row in the table as instances of the class """
        return cls.many_where()

    def delete(self):
        SQL = "DELETE FROM {tablename} WHERE pk=?"
        with sqlite3.connect(self.dbpath) as conn:
            cur = conn.cursor()
            cur.execute(SQL.format(tablename=self.dbtable), (self.pk,))
            self.pk = None

    def __repr__(self):
        reprstring = "<{cname} {fieldvals}>"
        # read over this:
        fieldvals = " ".join("{key}:{value}".format(key=key, value=getattr(self, key))
                             for key in ["pk", *self.fields])
        cname = type(self).__name__
        return reprstring.format(cname=cname, fieldvals=fieldvals)