text:
import random
import time
import warnings
import imghdr
import sndhdr
import logging
from collections import Counter
#import multiprocessing

import numpy as np
import scipy.stats as st
from dateutil.parser import parse as parse_datetime
from sklearn.metrics.pairwise import cosine_similarity
from sklearn.cluster import MiniBatchKMeans
import imagehash
from PIL import Image

from mindsdb.config import CONFIG
from mindsdb.libs.constants.mindsdb import *
from mindsdb.libs.phases.base_module import BaseModule
from mindsdb.libs.helpers.text_helpers import splitRecursive, clean_float, cast_string_to_python_type
from mindsdb.external_libs.stats import calculate_sample_size
from mindsdb.libs.phases.stats_generator.scores import *

class StatsGenerator(BaseModule):
    """
    # The stats generator phase is responsible for generating the insights we need about the data in order to vectorize it
    # Additionally, the stats generator also provides the user with some extra meaningful information about his data,
    though this functionality may be moved to a different step (after vectorization) in the future
    """

    phase_name = PHASE_STATS_GENERATOR

    def _get_file_type(self, potential_path):
        could_be_fp = False
        for char in ('/', '\\', ':\\'):
            if char in potential_path:
                could_be_fp = True

        if not could_be_fp:
            return False

        try:
            is_img = imghdr.what(potential_path)
            if is_img is not None:
                return DATA_SUBTYPES.IMAGE
        except:
            # Not a file or file doesn't exist
            return False

        # @TODO: CURRENTLY DOESN'T DIFFERENTIATE BETWEEN AUDIO AND VIDEO
        is_audio = sndhdr.what(potential_path)
        if is_audio is not None:
            return DATA_SUBTYPES.AUDIO

        return False

    def _is_number(self, string):
        """ Returns True if string is a number. """
        try:
            # Should crash if not number
            clean_float(str(string))
            if '.' in str(string) or ',' in str(string):
                return DATA_SUBTYPES.FLOAT
            else:
                return DATA_SUBTYPES.INT
        except ValueError:
            return False

    def _get_date_type(self, string):
        """ Returns True if string is a valid date format """
        try:
            dt = parse_datetime(string)

            # Not accurate 100% for a single datetime str, but should work in aggregate
            if dt.hour == 0 and dt.minute == 0 and dt.second == 0 and len(string) <= 16:
                return DATA_SUBTYPES.DATE
            else:
                return DATA_SUBTYPES.TIMESTAMP
        except:
            return False

    def _get_text_type(self, data):
        """
        Takes in column data and defines if its categorical or full_text

        :param data: a list of cells in a column
        :return: DATA_TYPES.CATEGORICAL or DATA_TYPES.FULL_TEXT
        """

        total_length = len(data)
        key_count = {}
        max_number_of_words = 0

        for cell in data:

            if cell not in key_count:
                key_count[cell] = 1
            else:
                key_count[cell] += 1

            cell_wseparator = cell
            sep_tag = '{#SEP#}'
            for separator in WORD_SEPARATORS:
                cell_wseparator = str(cell_wseparator).replace(separator,sep_tag)

            words_split = cell_wseparator.split(sep_tag)
            words = len([ word for word in words_split if word not in ['', None] ])

            if max_number_of_words < words:
                max_number_of_words += words

        # If all sentences are less than or equal and 3 words, assume it's a category rather than a sentence
        if max_number_of_words <= 3:
            if len(key_count.keys()) < 3:
                return DATA_TYPES.CATEGORICAL, DATA_SUBTYPES.SINGLE
            else:
                return DATA_TYPES.CATEGORICAL, DATA_SUBTYPES.MULTIPLE
        else:
            return DATA_TYPES.SEQUENTIAL, DATA_SUBTYPES.TEXT

    def _get_column_data_type(self, data, data_frame, col_name):
        """
        Provided the column data, define if its numeric, data or class

        :param data: a list containing each of the cells in a column

        :return: type and type distribution, we can later use type_distribution to determine data quality
        NOTE: type distribution is the count that this column has for belonging cells to each DATA_TYPE
        """

        type_dist = {}
        subtype_dist = {}
        additional_info = {}

        # calculate type_dist
        if len(data) < 1:
            self.log.warning(f'Column {col_name} has no data in it. Please remove {col_name} from the training file or fill in some of the values !')
            return None, None, None, None, None, 'Column empty'

        for element in data:
            # Maybe use list of functions in the future
            element = element
            current_subtype_guess = 'Unknown'
            current_type_guess = 'Unknown'

            # Check if Nr
            if current_subtype_guess is 'Unknown' or current_type_guess is 'Unknown':
                subtype = self._is_number(element)
                if subtype is not False:
                    current_type_guess = DATA_TYPES.NUMERIC
                    current_subtype_guess = subtype

            # Check if date
            if current_subtype_guess is 'Unknown' or current_type_guess is 'Unknown':
                subtype = self._get_date_type(element)
                if subtype is not False:
                    current_type_guess = DATA_TYPES.DATE
                    current_subtype_guess = subtype

            # Check if sequence
            if current_subtype_guess is 'Unknown' or current_type_guess is 'Unknown':
                for char in [',','\t','|',' ']:
                    try:
                        all_nr = True
                        eles = element.rstrip(']').lstrip('[').split(char)
                        for ele in eles:
                            if not self._is_number(ele):
                                all_nr = False
                    except:
                        all_nr = False
                        pass
                    if all_nr is True:
                        additional_info['separator'] = char
                        current_type_guess = DATA_TYPES.SEQUENTIAL
                        current_subtype_guess = DATA_SUBTYPES.ARRAY
                        break

            # Check if file
            if current_subtype_guess is 'Unknown' or current_type_guess is 'Unknown':
                subtype = self._get_file_type(element)
                if subtype is not False:
                    current_type_guess = DATA_TYPES.FILE_PATH
                    current_subtype_guess = subtype

            # If nothing works, assume it's categorical or sequential and determine type later (based on all the data in the column)

            if current_type_guess not in type_dist:
                type_dist[current_type_guess] = 1
            else:
                type_dist[current_type_guess] += 1

            if current_subtype_guess not in subtype_dist:
                subtype_dist[current_subtype_guess] = 1
            else:
                subtype_dist[current_subtype_guess] += 1

        curr_data_type = 'Unknown'
        curr_data_subtype = 'Unknown'
        max_data_type = 0

        # assume that the type is the one with the most prevalent type_dist
        for data_type in type_dist:
            # If any of the members are Unknown, use that data type (later to be turned into CATEGORICAL or SEQUENTIAL), since otherwise the model will crash when casting
            # @TODO consider removing rows where data type is unknown in the future, might just be corrupt data... a bit hard to imply currently
            if data_type == 'Unknown':
                curr_data_type = 'Unknown'
                break
            if type_dist[data_type] > max_data_type:
                curr_data_type = data_type
                max_data_type = type_dist[data_type]

        # Set subtype
        max_data_subtype = 0
        if curr_data_type != 'Unknown':
            for data_subtype in subtype_dist:
                if subtype_dist[data_subtype] > max_data_subtype and data_subtype in DATA_TYPES_SUBTYPES.subtypes[curr_data_type]:
                    curr_data_subtype = data_subtype
                    max_data_subtype = subtype_dist[data_subtype]

        # If it finds that the type is categorical it should determine if its categorical or actual text
        if curr_data_type == 'Unknown':
            curr_data_type, curr_data_subtype = self._get_text_type(data)
            type_dist[curr_data_type] = type_dist.pop('Unknown')
            subtype_dist[curr_data_subtype] = subtype_dist.pop('Unknown')

        # @TODO: Extremely slow for large datasets, make it faster
        if curr_data_type != DATA_TYPES.CATEGORICAL and curr_data_subtype != DATA_SUBTYPES.DATE:
            all_values = data_frame[col_name]
            all_distinct_vals = set(all_values)

            # The numbers here are picked randomly, the gist of it is that if values repeat themselves a lot we should consider the column to be categorical
            nr_vals = len(all_values)
            nr_distinct_vals = len(all_distinct_vals)

            if nr_vals/20 > nr_distinct_vals and (curr_data_type not in [DATA_TYPES.NUMERIC, DATA_TYPES.DATE] or nr_distinct_vals < 20) and nr_distinct_vals < 400:
                curr_data_type = DATA_TYPES.CATEGORICAL
                if len(all_distinct_vals) < 3:
                    curr_data_subtype = DATA_SUBTYPES.SINGLE
                else:
                    curr_data_subtype = DATA_SUBTYPES.MULTIPLE
                type_dist = {}
                subtype_dist = {}

                type_dist[curr_data_type] = len(data)
                subtype_dist[curr_data_subtype] = len(data)

        return curr_data_type, curr_data_subtype, type_dist, subtype_dist, additional_info, 'Column ok'

    @staticmethod
    def clean_int_and_date_data(col_data):
        cleaned_data = []

        for value in col_data:
            if value != '' and value != '\r' and value != '\n':
                cleaned_data.append(value)

        cleaned_data = [clean_float(i) for i in cleaned_data if str(i) not in ['', str(None), str(False), str(np.nan), 'NaN', 'nan', 'NA', 'null']]
        return cleaned_data

    @staticmethod
    def get_words_histogram(data, is_full_text=False):
        """ Returns an array of all the words that appear in the dataset and the number of times each word appears in the dataset """

        splitter = lambda w, t: [wi.split(t) for wi in w] if type(w) == type([]) else splitter(w,t)

        if is_full_text:
            # get all words in every cell and then calculate histograms
            words = []
            for cell in data:
                words += splitRecursive(cell, WORD_SEPARATORS)

            hist = {i: words.count(i) for i in words}
        else:
            hist = {i: data.count(i) for i in data}

        return {
            'x': list(hist.keys()),
            'y': list(hist.values())
        }

    @staticmethod
    def get_histogram(data, data_type=None, data_subtype=None, full_text=None, hmd=None):
        """ Returns a histogram for the data and [optionaly] the percentage buckets"""
        if data_type == DATA_TYPES.SEQUENTIAL:
            is_full_text = True if data_subtype == DATA_SUBTYPES.TEXT else False
            return StatsGenerator.get_words_histogram(data, is_full_text), None
        elif data_type == DATA_TYPES.NUMERIC or data_subtype == DATA_SUBTYPES.TIMESTAMP:
            data = StatsGenerator.clean_int_and_date_data(data)
            y, x = np.histogram(data, 50, density=False)
            x = (x + np.roll(x, -1))[:-1] / 2.0
            x = x.tolist()
            y = y.tolist()
            return {
                'x': x
                ,'y': y
            }, None
        elif data_type == DATA_TYPES.CATEGORICAL or data_subtype == DATA_SUBTYPES.DATE :
            histogram = Counter(data)
            return {
                'x': list(histogram.keys()),
                'y': list(histogram.values())
            }, None
        elif data_subtype == DATA_SUBTYPES.IMAGE:
            image_hashes = []
            for img_path in data:
                img_hash = imagehash.phash(Image.open(img_path))
                seq_hash = []
                for hash_row in img_hash.hash:
                    seq_hash.extend(hash_row)

                image_hashes.append(np.array(seq_hash))

            kmeans = MiniBatchKMeans(n_clusters=20, batch_size=round(len(image_hashes)/4))

            kmeans.fit(image_hashes)

            if hmd is not None:
                hmd['bucketing_algorithms'][col_name] = kmeans

            x = []
            y = [0] * len(kmeans.cluster_centers_)

            for cluster in kmeans.cluster_centers_:
                similarities = cosine_similarity(image_hashes,kmeans.cluster_centers_)

                similarities = list(map(lambda x: sum(x), similarities))

                index_of_most_similar = similarities.index(max(similarities))
                x.append(data.iloc[index_of_most_similar])

            indices = kmeans.predict(image_hashes)
            for index in indices:
                y[index] +=1

            return {
                'x': x,
                'y': y
            }, kmeans.cluster_centers_
        else:
            return None, None

    def _log_interesting_stats(self, stats):
        """
        # Provide interesting insights about the data to the user and send them to the logging server in order for it to generate charts

        :param stats: The stats extracted up until this point for all columns
        """
        for col_name in stats:
            col_stats = stats[col_name]
            # Overall quality
            if col_stats['quality_score'] < 6:
                # Some scores are not that useful on their own, so we should only warn users about them if overall quality is bad.
                self.log.warning('Column "{}" is considered of low quality, the scores that influenced this decision will be listed below')
                if 'duplicates_score' in col_stats and col_stats['duplicates_score'] < 6:
                    duplicates_percentage = col_stats['duplicates_percentage']
                    w = f'{duplicates_percentage}% of the values in column {col_name} seem to be repeated, this might indicate that your data is of poor quality.'
                    self.log.warning(w)
                    col_stats['duplicates_score_warning'] = w
                else:
                    col_stats['duplicates_score_warning'] = None
            else:
                col_stats['duplicates_score_warning'] = None

            #Compound scores
            if col_stats['consistency_score'] < 3:
                w = f'The values in column {col_name} rate poorly in terms of consistency. This means that the data has too many empty values, values with a hard to determine type and duplicate values. Please see the detailed logs below for more info'
                self.log.warning(w)
                col_stats['consistency_score_warning'] = w
            else:
                col_stats['consistency_score_warning'] = None

            if col_stats['redundancy_score'] < 5:
                w = f'The data in the column {col_name} is likely somewhat redundant, any insight it can give us can already by deduced from your other columns. Please see the detailed logs below for more info'
                self.log.warning(w)
                col_stats['redundancy_score_warning'] = w
            else:
                col_stats['redundancy_score_warning'] = None

            if col_stats['variability_score'] < 6:
                w = f'The data in the column {col_name} seems to contain too much noise/randomness based on the value variability. That is to say, the data is too unevenly distributed and has too many outliers. Please see the detailed logs below for more info.'
                self.log.warning(w)
                col_stats['variability_score_warning'] = w
            else:
                col_stats['variability_score_warning'] = None

            # Some scores are meaningful on their own, and the user should be warnned if they fall below a certain threshold
            if col_stats['empty_cells_score'] < 8:
                empty_cells_percentage = col_stats['empty_percentage']
                w = f'{empty_cells_percentage}% of the values in column {col_name} are empty, this might indicate that your data is of poor quality.'
                self.log.warning(w)
                col_stats['empty_cells_score_warning'] = w
            else:
                col_stats['empty_cells_score_warning'] = None

            if col_stats['data_type_distribution_score'] < 7:
                #self.log.infoChart(stats[col_name]['data_type_dist'], type='list', uid='Dubious Data Type Distribution for column "{}"'.format(col_name))
                percentage_of_data_not_of_principal_type = col_stats['data_type_distribution_score'] * 100
                principal_data_type = col_stats['data_type']
                w = f'{percentage_of_data_not_of_principal_type}% of your data is not of type {principal_data_type}, which was detected to be the data type for column {col_name}, this might indicate that your data is of poor quality.'
                self.log.warning(w)
                col_stats['data_type_distribution_score_warning'] = w
            else:
                col_stats['data_type_distribution_score_warning'] = None

            if 'z_test_based_outlier_score' in col_stats and col_stats['z_test_based_outlier_score'] < 6:
                percentage_of_outliers = col_stats['z_test_based_outlier_score']*100
                w = f"""Column {col_name} has a very high amount of outliers, {percentage_of_outliers}% of your data is more than 3 standard deviations away from the mean, this means that there might
                be too much randomness in this column for us to make an accurate prediction based on it."""
                self.log.warning(w)
                col_stats['z_test_based_outlier_score_warning'] = w
            else:
                col_stats['z_test_based_outlier_score_warning'] = None

            if 'lof_based_outlier_score' in col_stats and col_stats['lof_based_outlier_score'] < 4:
                percentage_of_outliers = col_stats['percentage_of_log_based_outliers']
                w = f"""Column {col_name} has a very high amount of outliers, {percentage_of_outliers}% of your data doesn't fit closely in any cluster using the KNN algorithm (20n) to cluster your data, this means that there might
                be too much randomness in this column for us to make an accurate prediction based on it."""
                self.log.warning(w)
                col_stats['lof_based_outlier_score_warning'] = w
            else:
                col_stats['lof_based_outlier_score_warning'] = None

            if col_stats['value_distribution_score'] < 3:
                max_probability_key = col_stats['max_probability_key']
                w = f"""Column {col_name} is very biased towards the value {max_probability_key}, please make sure that the data in this column is correct !"""
                self.log.warning(w)
                col_stats['value_distribution_score_warning'] = w
            else:
                col_stats['value_distribution_score_warning'] = None

            if col_stats['similarity_score'] < 6:
                similar_percentage = col_stats['max_similarity'] * 100
                similar_col_name = col_stats['most_similar_column_name']
                w = f'Column {col_name} and {similar_col_name} are {similar_percentage}% the same, please make sure these represent two distinct features of your data !'
                self.log.warning(w)
                col_stats['similarity_score_warning'] = w
            else:
                col_stats['similarity_score_warning'] = None

            '''
            if col_stats['correlation_score'] < 5:
                not_quite_correlation_percentage = col_stats['correlation_score'] * 100
                most_correlated_column = col_stats['most_correlated_column']
                self.log.warning(f"""Using a statistical predictor we\'v discovered a correlation of roughly {not_quite_correlation_percentage}% between column
                {col_name} and column {most_correlated_column}""")
            '''

            # We might want to inform the user about a few stats regarding his column regardless of the score, this is done below
            self.log.info('Data distribution for column "{}"'.format(col_name))
            self.log.infoChart(stats[col_name]['data_subtype_dist'], type='list', uid='Data Type Distribution for column "{}"'.format(col_name))

    def run(self, input_data, modify_light_metadata, hmd=None, print_logs=True):
        """
        # Runs the stats generation phase
        # This shouldn't alter the columns themselves, but rather provide the `stats` metadata object and update the types for each column
        # A lot of information about the data distribution and quality will  also be logged to the server in this phase
        """

        ''' @TODO Uncomment when we need multiprocessing, possibly disable on OSX
        no_processes = multiprocessing.cpu_count() - 2
        if no_processes < 1:
            no_processes = 1
        pool = multiprocessing.Pool(processes=no_processes)
        '''

        if print_logs == False:
            self.log = logging.getLogger('null-logger')
            self.log.propagate = False

        # we dont need to generate statistic over all of the data, so we subsample, based on our accepted margin of error
        population_size = len(input_data.data_frame)

        if population_size < 50:
            sample_size = population_size
        else:
            sample_size = int(calculate_sample_size(population_size=population_size, margin_error=self.transaction.lmd['sample_margin_of_error'], confidence_level=self.transaction.lmd['sample_confidence_level']))
            #if sample_size > 3000 and sample_size > population_size/8:
            #    sample_size = min(round(population_size/8),3000)

        # get the indexes of randomly selected rows given the population size
        input_data_sample_indexes = random.sample(range(population_size), sample_size)
        self.log.info('population_size={population_size},  sample_size={sample_size}  {percent:.2f}%'.format(population_size=population_size, sample_size=sample_size, percent=(sample_size/population_size)*100))

        all_sampled_data = input_data.data_frame.iloc[input_data_sample_indexes]

        stats = {}
        col_data_dict = {}

        for col_name in all_sampled_data.columns.values:
            col_data = all_sampled_data[col_name].dropna()
            full_col_data = all_sampled_data[col_name]

            data_type, curr_data_subtype, data_type_dist, data_subtype_dist, additional_info, column_status = self._get_column_data_type(col_data, input_data.data_frame, col_name)

            if column_status == 'Column empty':
                if modify_light_metadata:
                    self.transaction.lmd['malformed_columns']['names'].append(col_name)
                    self.transaction.lmd['malformed_columns']['indices'].append(i)
                continue

            new_col_data = []
            if curr_data_subtype == DATA_SUBTYPES.TIMESTAMP: #data_type == DATA_TYPES.DATE:
                for element in col_data:
                    if str(element) in [str(''), str(None), str(False), str(np.nan), 'NaN', 'nan', 'NA', 'null']:
                        new_col_data.append(None)
                    else:
                        try:
                            new_col_data.append(int(parse_datetime(element).timestamp()))
                        except:
                            self.log.warning(f'Could not convert string from col "{col_name}" to date and it was expected, instead got: {element}')
                            new_col_data.append(None)
                col_data = new_col_data
            if data_type == DATA_TYPES.NUMERIC or curr_data_subtype == DATA_SUBTYPES.TIMESTAMP:
                histogram, _ = StatsGenerator.get_histogram(col_data, data_type=data_type, data_subtype=curr_data_subtype)
                x = histogram['x']
                y = histogram['y']

                col_data = StatsGenerator.clean_int_and_date_data(col_data)
                # This means the column is all nulls, which we don't handle at the moment
                if len(col_data) < 1:
                    return None

                xp = []

                if len(col_data) > 0:
                    max_value = max(col_data)
                    min_value = min(col_data)
                    mean = np.mean(col_data)
                    median = np.median(col_data)
                    var = np.var(col_data)
                    skew = st.skew(col_data)
                    kurtosis = st.kurtosis(col_data)

                    inc_rate = 0.1
                    initial_step_size = abs(max_value-min_value)/100

                    xp += [min_value]
                    i = min_value + initial_step_size

                    while i < max_value:

                        xp += [i]
                        i_inc = abs(i-min_value)*inc_rate
                        i = i + i_inc
                else:
                    max_value = 0
                    min_value = 0
                    mean = 0
                    median = 0
                    var = 0
                    skew = 0
                    kurtosis = 0
                    xp = []

                is_float = True if max([1 if int(i) != i else 0 for i in col_data]) == 1 else False

                col_stats = {
                    'data_type': data_type,
                    'data_subtype': curr_data_subtype,
                    "mean": mean,
                    "median": median,
                    "variance": var,
                    "skewness": skew,
                    "kurtosis": kurtosis,
                    "max": max_value,
                    "min": min_value,
                    "is_float": is_float,
                    "histogram": {
                        "x": x,
                        "y": y
                    },
                    "percentage_buckets": xp
                }
            elif data_type == DATA_TYPES.CATEGORICAL or curr_data_subtype == DATA_SUBTYPES.DATE:
                histogram, _ = StatsGenerator.get_histogram(input_data.data_frame[col_name], data_type=data_type, data_subtype=curr_data_subtype)

                col_stats = {
                    'data_type': data_type,
                    'data_subtype': curr_data_subtype,
                    "histogram": histogram,
                    "percentage_buckets": histogram['x']
                }

            elif curr_data_subtype == DATA_SUBTYPES.IMAGE:
                histogram, percentage_buckets = StatsGenerator.get_histogram(col_data, data_subtype=curr_data_subtype)

                col_stats = {
                    'data_type': data_type,
                    'data_subtype': curr_data_subtype,
                    'percentage_buckets': percentage_buckets,
                    'histogram': histogram
                }

            # @TODO This is probably wrong, look into it a bit later
            else:
                # see if its a sentence or a word
                histogram, _ = StatsGenerator.get_histogram(col_data, data_type=data_type, data_subtype=curr_data_subtype)
                dictionary = list(histogram.keys())

                # if no words, then no dictionary
                if len(col_data) == 0:
                    dictionary_available = False
                    dictionary_lenght_percentage = 0
                    dictionary = []
                else:
                    dictionary_available = True
                    dictionary_lenght_percentage = len(
                        dictionary) / len(col_data) * 100
                    # if the number of uniques is too large then treat is a text
                    is_full_text = True if curr_data_subtype == DATA_SUBTYPES.TEXT else False
                    if dictionary_lenght_percentage > 10 and len(col_data) > 50 and is_full_text==False:
                        dictionary = []
                        dictionary_available = False

                col_stats = {
                    'data_type': data_type,
                    'data_subtype': curr_data_subtype,
                    "dictionary": dictionary,
                    "dictionaryAvailable": dictionary_available,
                    "dictionaryLenghtPercentage": dictionary_lenght_percentage,
                    "histogram": histogram
                }
            stats[col_name] = col_stats
            stats[col_name]['data_type_dist'] = data_type_dist
            stats[col_name]['data_subtype_dist'] = data_subtype_dist
            stats[col_name]['column'] = col_name

            empty_count = len(full_col_data) - len(col_data)

            stats[col_name]['empty_cells'] = empty_count
            stats[col_name]['empty_percentage'] = empty_count * 100 / len(full_col_data)
            if 'separator' in additional_info:
                stats[col_name]['separator'] = additional_info['separator']
            col_data_dict[col_name] = col_data

        for col_name in all_sampled_data.columns:
            if col_name in self.transaction.lmd['malformed_columns']['names']:
                continue

            # Use the multiprocessing pool for computing scores which take a very long time to compute
            # For now there's only one and computing it takes way too long, so this is not enabled
            scores = []

            '''
            scores.append(pool.apply_async(compute_clf_based_correlation_score, args=(stats, all_sampled_data, col_name)))
            '''
            for score_promise in scores:
                # Wait for function on process to finish running
                score = score_promise.get()
                stats[col_name].update(score)

            for score_func in [compute_duplicates_score, compute_empty_cells_score, compute_data_type_dist_score, compute_z_score, compute_lof_score, compute_similariy_score, compute_value_distribution_score]:
                start_time = time.time()
                if 'compute_z_score' in str(score_func) or 'compute_lof_score' in str(score_func):
                    stats[col_name].update(score_func(stats, col_data_dict, col_name))
                else:
                    stats[col_name].update(score_func(stats, all_sampled_data, col_name))

                fun_name = str(score_func)
                run_duration = round(time.time() - start_time, 2)
                #print(f'Running scoring function "{run_duration}" took {run_duration} seconds !')

            stats[col_name].update(compute_consistency_score(stats, col_name))
            stats[col_name].update(compute_redundancy_score(stats, col_name))
            stats[col_name].update(compute_variability_score(stats, col_name))

            stats[col_name].update(compute_data_quality_score(stats, col_name))

        total_rows = len(input_data.data_frame)

        if modify_light_metadata:
            self.transaction.lmd['column_stats'] = stats

            self.transaction.lmd['data_preparation']['accepted_margin_of_error'] = self.transaction.lmd['sample_margin_of_error']

            self.transaction.lmd['data_preparation']['total_row_count'] = total_rows
            self.transaction.lmd['data_preparation']['used_row_count'] = sample_size
            self.transaction.lmd['data_preparation']['test_row_count'] = len(input_data.test_indexes[KEY_NO_GROUP_BY])
            self.transaction.lmd['data_preparation']['train_row_count'] = len(input_data.train_indexes[KEY_NO_GROUP_BY])
            self.transaction.lmd['data_preparation']['validation_row_count'] = len(input_data.validation_indexes[KEY_NO_GROUP_BY])

        ''' @TODO Uncomment when we need multiprocessing, possibly disable on OSX
        pool.close()
        pool.join()
        '''

        self._log_interesting_stats(stats)

        return stats

max_stars_repo_name: mohsinkhansymc/mindsdb

max_issues_repo_name: mohsinkhansymc/mindsdb

max_forks_repo_name: mohsinkhansymc/mindsdb

reason: import numpy,import scipy


text:
import argparse
import json

import numpy as np
import pandas as pd
import yaml
from tensorflow.keras.applications.mobilenet_v2 import preprocess_input
from tensorflow.keras.models import load_model

from preprocessing_utilities import (
    read_img_from_path,
    resize_img,
    read_from_file,
)
from utils import download_model

class ImagePredictor:
    def __init__(
        self,
        model_path,
        resize_size,
        class_display_names,
        pre_processing_function=preprocess_input,
    ):
        self.model_path = model_path
        self.pre_processing_function = pre_processing_function
        self.model = load_model(self.model_path, compile=False)
        self.resize_size = resize_size
        self.class_display_names = class_display_names

    @classmethod
    def init_from_config_path(cls, config_path):
        with open(config_path, "r") as f:
            config = yaml.load(f, yaml.SafeLoader)
        predictor = cls(
            model_path=config["model_path"],
            resize_size=config["resize_shape"],
            class_display_names=config["class_display_names"],
        )
        return predictor

    @classmethod
    def init_from_config_url(cls, config_path):
        with open(config_path, "r") as f:
            config = yaml.load(f, yaml.SafeLoader)

        download_model(
            config["model_url"], config["model_path"], config["model_sha256"],
        )

        return cls.init_from_config_path(config_path)

    def predict_from_array(self, arr):
        arr = resize_img(arr, h=self.resize_size[0], w=self.resize_size[1])
        arr = self.pre_processing_function(arr)
        pred = self.model.predict(arr[np.newaxis, ...]).ravel()
        label, max_score = np.argmax(pred), np.max(pred)
        label_name = (
            self.class_display_names[label] if max_score > 0.15 else "Don't Know"
        )
        # label = label + 1  # labels are 1-indexed while arrays are 0-indexed ..;
        return {"label": label_name, "score": float(max_score)}

    def predict_from_path(self, path):
        arr = read_img_from_path(path)
        return self.predict_from_array(arr)

    def predict_from_file(self, file_object):
        arr = read_from_file(file_object)
        return self.predict_from_array(arr), arr

if __name__ == "__main__":
    """
    python predictor.py --predictor_config "config.yaml"

    """
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--predictor_config_path", help="predictor_config_path", default="config.yaml",
    )

    args = parser.parse_args()

    predictor_config_path = args.predictor_config_path

    predictor = ImagePredictor.init_from_config_url(predictor_config_path)

    pred = predictor.predict_from_file(open("../example/data/image_00005.jpg", "rb"))

    print(pred)

ext: py

reason: import numpy


text:
import numpy as np
import pandas as pd
import joblib
import argparse
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.neural_network import MLPClassifier
from IPython.display import display

SIZE_OF_SAMPLE = 5000

def main():
    fields = ['sha256','strings', 'imports', 'size', 'signed', 'good_file', 'entropy']

    parser = argparse.ArgumentParser(description="File Indexer for Blue Sentry")
    parser.add_argument("--data_file", help="File that the data is output to in csv form", required=True)
    parser.add_argument("--imports_file", help="File corresponding to the imports of the data being classified",
                        required=True)
    parser.add_argument("--strings_file", help="File corresponding to the imports of the data being classified",
                        required=True)
    parser.add_argument("--classifier_imports", help="The file containing the saved imports classifier", required=True)
    parser.add_argument("--classifier_strings", help="The file containing the saved strings classifier", required=True)
    parser.add_argument("--data_out_file", help="File to add the estimations to", required=True)
    args = parser.parse_args()

    args = parser.parse_args()
    # import some data to play with
    to_test = range(50000)
    df_parser = pd.read_csv(args.data_file, chunksize=SIZE_OF_SAMPLE, iterator=True, skipinitialspace=True)

    text_encoding_imports = open(args.imports_file, 'r').read()
    text_encoding_strings = open(args.strings_file, 'r').read()
    encoder_imports = CountVectorizer().fit(text_encoding_imports.split())
    encoder_strings = CountVectorizer().fit(text_encoding_strings.split())
    clf_imports = joblib.load(args.classifier_imports)
    clf_strings = joblib.load(args.classifier_strings)

    output_file = open(args.data_out_file, 'a')

    first_file = True
    current_index = 0
    for df in df_parser:
        df.columns = fields
        data_imports_encoded = encoder_imports.transform(df['imports'])
        data_strings_encoded = encoder_strings.transform(df['strings'])
        pred_prob_imports = clf_imports.predict_proba(data_imports_encoded)
        pred_prob_strings = clf_strings.predict_proba(data_strings_encoded)
        pred_prob_rounded_imports = []
        for x in range(0, data_imports_encoded.shape[0]):
            pred_prob_rounded_imports.append(round(pred_prob_imports[x][1], 8))

        df['imports_processed'] = pred_prob_rounded_imports

        pred_prob_rounded_strings = []
        for x in range(0, data_strings_encoded.shape[0]):
            pred_prob_rounded_strings.append(round(pred_prob_strings[x][1], 8))

        df['strings_processed'] = pred_prob_rounded_strings
        df = df.drop(['strings'], axis=1)
        df = df.drop(['imports'], axis=1)
        print df

        if first_file:
            df.to_csv(output_file, header=True, index=False, index_label=False)
            first_file = False
        else:
            df.to_csv(output_file, header=False, index=False, index_label=False)
        current_index += 5000
        print current_index
    output_file.close()
'''
    data[args.data_column_name] = pred_prob_rounded
    data = data[['sha256', args.data_column_name, 'imports', 'signed', 'entropy', 'size', 'good_file']]

    data.to_csv(args.data_out_file, index=False, index_label=False)
    display(data)
'''

if __name__ == "__main__":
    main()

max_stars_repo_name: chrisbvt/ML-MalwareDetection

max_issues_repo_name: chrisbvt/ML-MalwareDetection

max_forks_repo_name: chrisbvt/ML-MalwareDetection


text:
# -*- coding: utf-8 -*-
"""
Created on Fri Mar  3 14:15:30 2017

@author: DamianCristian
"""

import numpy as np
from scipy import ndimage as ndi
from scipy import signal

sigm = lambda x, w: -np.where(abs(x)<w/2, 2*x, np.sign(x))/2 + .5
scale = lambda x: (x-x.min())/np.sum(x-x.min())

def psf_mesh(dim):
    w = 2/abs(dim)
    x = np.arange(w,1+w,w)
    x = np.concatenate((-x[::-1],[0],x[::1]))
    x, y = np.meshgrid(x,x)
    return x, y, w

def pillbox_psf(dim):
    x, y, w = psf_mesh(dim)
    h = sigm((x**2 + y**2)**0.5 - 1, w)
    return h

def box_psf(dim):
    x, y, w = psf_mesh(dim)
    h = sigm(np.maximum(abs(x)+ abs(y)) - 1, w)
    return h

class Coded_psf:
    """ The class that models a coded aperture. It holds a matrix describing
    the aperture and when called returns a PSF of the given size.

    Parameters
    -----------

    prototype: array like
        A matrix holding the transmissivity profile of the aperture.

    """

    def __init__(self, prototype):
        self.mat = scale(np.array(prototype, dtype= float))
        self.s = max(self.mat.shape)

    def __call__(self, d):
        sfact = self.s//d + 1
        #w =  np.bartlett(2*sfact+1)
        w = signal.bspline(np.linspace(-2,2,4*sfact+1),3)
        fh = ndi.convolve1d(self.mat, w, axis=0, mode='constant')
        fh = ndi.convolve1d(fh, w, axis= 1, mode='constant')
        sh = ndi.zoom( fh, d/self.s, order=3, prefilter=False)
        return sh

class Camera:
    """This class holds the characteristics of the camera.
    And performs basic calculations such as computing the PSF according to distance. It uses a thin lens model for computation.

    By convention all lengths are expressed in millimeters.

    Parameters
    -----------

    focal_length: float
        The focal length of the lens.

    f_stop: float
        The ratio of focal length over the diameter of the lens.

    pixel_pitch: float
        The distance between the centers of two adjacent pixels.

    aperture:
        The shape of the aperture of the camera.

    focus:
        The distance at witch the camera is focused.

    Example
    --------
    ::

        testCam = de.Camera(focal_length = 105.0,
                      f_stop = 4.0,
                      pixel_pitch = 0.0082)
        testCam.set_focal_plane(1600)
        print('Psf diameter:', testCam.d_psf(1700))
        print('Psf')
        print(testCam.psf(1700))

    """

    def __init__(self,
                 focal_length,
                 f_stop,
                 pixel_pitch,
                 aperture=None,
                 focus=np.inf):

        self.fL = focal_length
        self.fN = f_stop
        self.pitch = pixel_pitch
        self.focus = focus

        if aperture is None:
            self.kernel = pillbox_psf
        else:
            self.kernel = aperture

    def set_focal_plane(self, distance):
        "Sets the focal plane of the camera."
        self.focus = distance

    def set_f_stop(self, value):
        self.fN = value

    @property
    def aperture_diameter(self):
        "Diameter of the camera's aperture."
        return self.fL/self.fN

    def psf_diameter(self,distance):
        "Returns the diameter of the psf given the distance to the scene."
        fL = self.fL
        focus = self.focus
        V = 1/(1/fL - 1/focus)
        diam = fL / self.fN
        return diam*V*(1/distance-1/focus)

    def distance(self, diam_psf):
        "Returns the distance of the object if given the diameter of the PSF."
        fL = self.fL
        focus = self.focus
        s = 1/(1/fL - 1/focus)
        diam = fL / self.fN
        return s/(s/fL - diam_psf/diam -1)

    def set_aperture(self, fun):
        "Sets the shape of the aperture for the camera."
        self.kernel = fun

    def psf(self, distance= None, diameter=None):
        "Returns the PSF given the distance to the imaged object or the diameter of the PSF."

        if distance is None and diameter is None:
            raise TypeError("At least one parameter must be defined.")

        if not distance is None:
            pitch = self.pitch
            psfd = self.psf_diameter(distance)
            kerd = abs(psfd/pitch)
        else:
            kerd = diameter

        if kerd < 1:
            return np.array(1, ndmin = 2)

        h = self.kernel(kerd)
        return h/np.sum(h)

max_stars_repo_name: CrisDamian/BlurEqualisationTechnique

max_issues_repo_name: CrisDamian/BlurEqualisationTechnique

max_forks_repo_name: CrisDamian/BlurEqualisationTechnique

reason: import numpy,from scipy


text:
import matplotlib as mpl

# This line allows mpl to run with no DISPLAY defined
mpl.use('Agg')

import pandas as pd
import numpy as np
import os
from keras.layers import Reshape, Flatten, LeakyReLU, Activation
from keras.models import Sequential
from keras.optimizers import Adam
from keras.callbacks import TensorBoard
from keras_adversarial.image_grid_callback import ImageGridCallback
from keras_adversarial import AdversarialModel, simple_gan, gan_targets
from keras_adversarial import normal_latent_sampling, AdversarialOptimizerSimultaneous
from keras_adversarial.legacy import l1l2, Dense, fit
import keras.backend as K
from mnist_utils import mnist_data

def model_generator(latent_dim, input_shape, hidden_dim=1024, reg=lambda: l1l2(1e-5, 1e-5)):
    return Sequential([
        Dense(int(hidden_dim / 4), name="generator_h1", input_dim=latent_dim, W_regularizer=reg()),
        LeakyReLU(0.2),
        Dense(int(hidden_dim / 2), name="generator_h2", W_regularizer=reg()),
        LeakyReLU(0.2),
        Dense(hidden_dim, name="generator_h3", W_regularizer=reg()),
        LeakyReLU(0.2),
        Dense(np.prod(input_shape), name="generator_x_flat", W_regularizer=reg()),
        Activation('sigmoid'),
        Reshape(input_shape, name="generator_x")],
        name="generator")

def model_discriminator(input_shape, hidden_dim=1024, reg=lambda: l1l2(1e-5, 1e-5), output_activation="sigmoid"):
    return Sequential([
        Flatten(name="discriminator_flatten", input_shape=input_shape),
        Dense(hidden_dim, name="discriminator_h1", W_regularizer=reg()),
        LeakyReLU(0.2),
        Dense(int(hidden_dim / 2), name="discriminator_h2", W_regularizer=reg()),
        LeakyReLU(0.2),
        Dense(int(hidden_dim / 4), name="discriminator_h3", W_regularizer=reg()),
        LeakyReLU(0.2),
        Dense(1, name="discriminator_y", W_regularizer=reg()),
        Activation(output_activation)],
        name="discriminator")

def example_gan(adversarial_optimizer, path, opt_g, opt_d, nb_epoch, generator, discriminator, latent_dim,
                targets=gan_targets, loss='binary_crossentropy'):
    csvpath = os.path.join(path, "history.csv")
    if os.path.exists(csvpath):
        print("Already exists: {}".format(csvpath))
        return

    print("Training: {}".format(csvpath))
    # gan (x - > yfake, yreal), z generated on GPU
    gan = simple_gan(generator, discriminator, normal_latent_sampling((latent_dim,)))

    # print summary of models
    generator.summary()
    discriminator.summary()
    gan.summary()

    # build adversarial model
    model = AdversarialModel(base_model=gan,
                             player_params=[generator.trainable_weights, discriminator.trainable_weights],
                             player_names=["generator", "discriminator"])
    model.adversarial_compile(adversarial_optimizer=adversarial_optimizer,
                              player_optimizers=[opt_g, opt_d],
                              loss=loss)

    # create callback to generate images
    zsamples = np.random.normal(size=(10 * 10, latent_dim))

    def generator_sampler():
        return generator.predict(zsamples).reshape((10, 10, 28, 28))

    generator_cb = ImageGridCallback(os.path.join(path, "epoch-{:03d}.png"), generator_sampler)

    # train model
    xtrain, xtest = mnist_data()
    y = targets(xtrain.shape[0])
    ytest = targets(xtest.shape[0])
    callbacks = [generator_cb]
    if K.backend() == "tensorflow":
        callbacks.append(
            TensorBoard(log_dir=os.path.join(path, 'logs'), histogram_freq=0, write_graph=True, write_images=True))
    history = fit(model, x=xtrain, y=y, validation_data=(xtest, ytest), callbacks=callbacks, nb_epoch=nb_epoch,
                  batch_size=32)

    # save history to CSV
    df = pd.DataFrame(history.history)
    df.to_csv(csvpath)

    # save models
    generator.save(os.path.join(path, "generator.h5"))
    discriminator.save(os.path.join(path, "discriminator.h5"))

def main():
    # z \in R^100
    latent_dim = 100
    # x \in R^{28x28}
    input_shape = (28, 28)
    # generator (z -> x)
    generator = model_generator(latent_dim, input_shape)
    # discriminator (x -> y)
    discriminator = model_discriminator(input_shape)
    example_gan(AdversarialOptimizerSimultaneous(), "output/gan",
                opt_g=Adam(1e-4, decay=1e-4),
                opt_d=Adam(1e-3, decay=1e-4),
                nb_epoch=100, generator=generator, discriminator=discriminator,
                latent_dim=latent_dim)

if __name__ == "__main__":
    main()

max_issues_repo_name: tungsomot/keras-adversarial

max_forks_repo_name: tungsomot/keras-adversarial


text:
"""Water network transfers maps
"""
import os
import sys
from collections import OrderedDict

import numpy as np
import geopandas as gpd
import pandas as pd
import cartopy.crs as ccrs
import cartopy.io.shapereader as shpreader
import matplotlib.pyplot as plt
import numpy as np
from shapely.geometry import LineString
from vtra.utils import *

def main(mode):
    config = load_config()
    if mode == 'road':
        flow_file_path = os.path.join(config['paths']['output'], 'failure_results','minmax_combined_scenarios',
                               'single_edge_failures_transfers_national_road_10_percent_shift.csv')
    elif mode == 'rail':
        flow_file_path = os.path.join(config['paths']['output'], 'failure_results','minmax_combined_scenarios',
                                   'single_edge_failures_transfers_national_rail_100_percent_shift.csv')
    else:
        raise ValueError("Mode must be road or rail")

    region_file_path = os.path.join(config['paths']['data'], 'post_processed_networks',
                               'coastal_edges.shp')

    region_file = gpd.read_file(region_file_path,encoding='utf-8')
    flow_file = pd.read_csv(flow_file_path)
    region_file = pd.merge(region_file,flow_file,how='left', on=['edge_id']).fillna(0)
    del flow_file

    region_file = region_file[region_file['edge_id'] != 0]

    color = '#045a8d'
    color_by_type = {'Coastal Line': color}

    columns = ['min_tons', 'max_tons']
    column_label_divisors = {c: 1 for c in columns}

    legend_label = "AADF (tons/day)"
    title_cols = ['Total tonnage (min)', 'Total tonnage (max)']

    for c in range(len(columns)):
        ax = get_axes()
        plot_basemap(ax, config['paths']['data'], highlight_region=[])
        scale_bar(ax, location=(0.8, 0.05))
        plot_basemap_labels(ax, config['paths']['data'],plot_international_left=False)
        proj_lat_lon = ccrs.PlateCarree()

        column = columns[c]
        weights = [
            record[column]
            for iter_,record in region_file.iterrows()
        ]
        max_weight = max(weights)
        width_by_range = generate_weight_bins(weights)

        water_geoms_by_category = {
            '1': [],
            '2': []
        }

        geoms_by_range = {}
        for value_range in width_by_range:
            geoms_by_range[value_range] = []

        for iter_,record in region_file.iterrows():
            geom = record.geometry
            val = record[column]
            if val == 0:
                cat = '2'
            else:
                cat = '1'

            buffered_geom = None
            for (nmin, nmax), width in width_by_range.items():
                if nmin <= val and val < nmax:
                    buffered_geom = geom.buffer(width)

            if buffered_geom is not None:
                water_geoms_by_category[cat].append(buffered_geom)
            else:
                print("Feature was outside range to plot", iter_)

        styles = OrderedDict([
            ('1',  Style(color='#045a8d', zindex=9, label='Transfers')),  # blue
            ('2', Style(color='#969696', zindex=7, label='No transfer'))
        ])

        for cat, geoms in water_geoms_by_category.items():
            cat_style = styles[cat]
            ax.add_geometries(
                geoms,
                crs=proj_lat_lon,
                linewidth=0,
                facecolor=cat_style.color,
                edgecolor='none',
                zorder=cat_style.zindex
            )

        x_l = 102.3
        x_r = x_l + 0.4
        base_y = 14
        y_step = 0.4
        y_text_nudge = 0.1
        x_text_nudge = 0.1

        ax.text(
            x_l,
            base_y + y_step - y_text_nudge,
            legend_label,
            horizontalalignment='left',
            transform=proj_lat_lon,
            size=10)

        divisor = column_label_divisors[column]
        for (i, ((nmin, nmax), width)) in enumerate(width_by_range.items()):
            y = base_y - (i*y_step)
            line = LineString([(x_l, y), (x_r, y)])
            ax.add_geometries(
                [line.buffer(width)],
                crs=proj_lat_lon,
                linewidth=0,
                edgecolor=color,
                facecolor=color,
                zorder=2)
            if nmin == max_weight:
                label = '>{:.2f}'.format(max_weight/divisor)
            else:
                label = '{:.2f}-{:.2f}'.format(nmin/divisor, nmax/divisor)
            ax.text(
                x_r + x_text_nudge,
                y - y_text_nudge,
                label,
                horizontalalignment='left',
                transform=proj_lat_lon,
                size=10)

        plt.title('{} transfers - '.format(mode.title()) + title_cols[c], fontsize=14)
        legend_from_style_spec(ax, styles)
        print ('* Plotting ',title_cols[c])
        if mode == 'road':
            output_file = os.path.join(
                config['paths']['figures'], 'water_flow-map-transfer-{}-10-shift-{}.png'.format(mode,column))
        elif mode == 'rail':
            output_file = os.path.join(
                config['paths']['figures'], 'water_flow-map-transfer-{}-100-shift-{}.png'.format(mode,column))
        else:
            raise ValueError("Mode must be road or rail")
        save_fig(output_file)
        plt.close()

if __name__ == '__main__':
    ok_values = ('road', 'rail')
    for ok in ok_values:
        main(ok)

max_stars_repo_name: GFDRR/vietnam-transport

max_issues_repo_name: GFDRR/vietnam-transport

max_forks_repo_name: GFDRR/vietnam-transport


text:
"""
python make_attribute_vocab.py [vocab] [corpus1] [corpus2] r
subsets a [vocab] file by finding the words most associated with
one of two corpuses. threshold is r ( # in corpus_a  / # in corpus_b )
"""
import sys
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
import numpy as np

class SalienceCalculator(object):
    def __init__(self, pre_corpus, post_corpus):
        self.vectorizer = CountVectorizer()

        pre_count_matrix = self.vectorizer.fit_transform(pre_corpus)
        self.pre_vocab = self.vectorizer.vocabulary_
        self.pre_counts = np.sum(pre_count_matrix, axis=0)
        self.pre_counts = np.squeeze(np.asarray(self.pre_counts))

        post_count_matrix = self.vectorizer.fit_transform(post_corpus)
        self.post_vocab = self.vectorizer.vocabulary_
        self.post_counts = np.sum(post_count_matrix, axis=0)
        self.post_counts = np.squeeze(np.asarray(self.post_counts))

    def salience(self, feature, attribute='pre', lmbda=0.5):
        assert attribute in ['pre', 'post']

        if feature not in self.pre_vocab:
            pre_count = 0.0
        else:
            pre_count = self.pre_counts[self.pre_vocab[feature]]

        if feature not in self.post_vocab:
            post_count = 0.0
        else:
            post_count = self.post_counts[self.post_vocab[feature]]

        if attribute == 'pre':
            return (pre_count + lmbda) / (post_count + lmbda)
        else:
            return (post_count + lmbda) / (pre_count + lmbda)

vocab = set([w.strip() for i, w in enumerate(open(sys.argv[1]))])
corpus1 = sys.argv[2]
corpus1 = [
    w if w in vocab else '<unk>'
    for l in open(corpus1)
    for w in l.strip().split()
]

corpus2 = sys.argv[3]
corpus2 = [
    w if w in vocab else '<unk>'
    for l in open(corpus2)
    for w in l.strip().split()
]

r = float(sys.argv[4])
lmbda = float(sys.argv[5])

sc = SalienceCalculator(corpus1, corpus2)

for tok in vocab:
#    print(tok, sc.salience(tok))
    if max(sc.salience(tok, attribute='pre'), sc.salience(tok, attribute='post'), lmbda) > r:
        print(tok)

max_stars_repo_name: akhisud/pytorch-pretrained-BERT

max_issues_repo_name: akhisud/pytorch-pretrained-BERT

max_forks_repo_name: akhisud/pytorch-pretrained-BERT


text:
import copy
import itertools

import numpy as np
import pytest
import torch

from composer.algorithms import EMAHparams
from composer.algorithms.ema.ema import ShadowModel, compute_ema
from composer.core import Event, Time, Timer, TimeUnit
from tests.common import SimpleConvModel, SimpleModel

def validate_ema(model, original_model, ema_model, smoothing):
    model_params = itertools.chain(model.parameters(), model.buffers())
    original_params = itertools.chain(original_model.parameters(), original_model.buffers())
    ema_params = itertools.chain(ema_model.parameters(), ema_model.buffers())

    for model_param, original_param, ema_param in zip(model_params, original_params, ema_params):
        new_param = original_param * smoothing + (1. - smoothing) * model_param
        torch.testing.assert_allclose(ema_param, new_param)

def validate_model(model1, model2):
    model1_params = itertools.chain(model1.parameters(), model1.buffers())
    model2_params = itertools.chain(model2.parameters(), model2.buffers())

    for model1_param, model2_param in zip(model1_params, model2_params):
        torch.testing.assert_allclose(model1_param, model2_param)

@pytest.mark.parametrize("smoothing", [0, 0.5, 0.99, 1])
def test_ema(smoothing):
    model = SimpleModel()
    ema_model = SimpleModel()
    original_model = copy.deepcopy(ema_model)
    compute_ema(model=model, ema_model=ema_model, smoothing=smoothing)
    validate_ema(model, original_model, ema_model, smoothing)

# params = [(half_life, update_interval)]
@pytest.mark.parametrize('params', [("10ba", "1ba"), ("1ep", "1ep")])
def test_ema_algorithm(params, minimal_state, empty_logger):

    # Initialize input tensor
    input = torch.rand((32, 5))

    half_life, update_interval = params[0], params[1]
    algorithm = EMAHparams(half_life=half_life, update_interval=update_interval,
                           train_with_ema_weights=False).initialize_object()
    state = minimal_state
    state.model = SimpleConvModel()
    state.batch = (input, torch.Tensor())

    # Start EMA
    algorithm.ema_model = ShadowModel(state.model)
    algorithm.training_model = ShadowModel(state.model)
    # Check if ema correctly calculated smoothing
    half_life = Time.from_timestring(params[0])
    update_interval = Time.from_timestring(params[1])
    smoothing = np.exp(-np.log(2) * (update_interval.value / half_life.value))
    np.testing.assert_allclose(smoothing, algorithm.smoothing)

    # Fake a training update by replacing state.model after ema grabbed it.
    original_model = copy.deepcopy(state.model)
    state.model = SimpleConvModel()
    # Do the EMA update
    state.timer = Timer()
    if half_life.unit == TimeUnit.BATCH:
        state.timer._batch = update_interval
        algorithm.apply(Event.BATCH_END, state, empty_logger)
    elif half_life.unit == TimeUnit.EPOCH:
        state.timer._epoch = update_interval
        algorithm.apply(Event.EPOCH_END, state, empty_logger)
    else:
        raise ValueError(f"Invalid time string for parameter half_life")
    # Check if EMA correctly computed the average.
    validate_ema(state.model, original_model, algorithm.ema_model, algorithm.smoothing)
    # Check if the EMA model is swapped in for testing
    algorithm.apply(Event.EVAL_START, state, empty_logger)
    validate_model(state.model, algorithm.ema_model)
    # Check if the training model is swapped back in for training
    algorithm.apply(Event.EVAL_END, state, empty_logger)
    validate_model(state.model, algorithm.training_model)
    # Check if the ema model can be extracted correctly
    overwrite_model = copy.deepcopy(original_model)
    overwrite_model = algorithm.get_ema_model(overwrite_model)
    validate_model(overwrite_model, algorithm.ema_model)

max_stars_repo_name: ravi-mosaicml/ravi-composer

max_issues_repo_name: ravi-mosaicml/ravi-composer

max_forks_repo_name: ravi-mosaicml/ravi-composer


text:
'''
               AAA               lllllll lllllll   iiii
              A:::A              l:::::l l:::::l  i::::i
             A:::::A             l:::::l l:::::l   iiii
            A:::::::A            l:::::l l:::::l
           A:::::::::A            l::::l  l::::l iiiiiii     eeeeeeeeeeee
          A:::::A:::::A           l::::l  l::::l i:::::i   ee::::::::::::ee
         A:::::A A:::::A          l::::l  l::::l  i::::i  e::::::eeeee:::::ee
        A:::::A   A:::::A         l::::l  l::::l  i::::i e::::::e     e:::::e
       A:::::A     A:::::A        l::::l  l::::l  i::::i e:::::::eeeee::::::e
      A:::::AAAAAAAAA:::::A       l::::l  l::::l  i::::i e:::::::::::::::::e
     A:::::::::::::::::::::A      l::::l  l::::l  i::::i e::::::eeeeeeeeeee
    A:::::AAAAAAAAAAAAA:::::A     l::::l  l::::l  i::::i e:::::::e
   A:::::A             A:::::A   l::::::ll::::::li::::::ie::::::::e
  A:::::A               A:::::A  l::::::ll::::::li::::::i e::::::::eeeeeeee
 A:::::A                 A:::::A l::::::ll::::::li::::::i  ee:::::::::::::e
AAAAAAA                   AAAAAAAlllllllllllllllliiiiiiii    eeeeeeeeeeeeee

|  ___|       | |                        / _ \ | ___ \_   _|  _
| |_ ___  __ _| |_ _   _ _ __ ___  ___  / /_\ \| |_/ / | |   (_)
|  _/ _ \/ _` | __| | | | '__/ _ \/ __| |  _  ||  __/  | |
| ||  __/ (_| | |_| |_| | | |  __/\__ \ | | | || |    _| |_   _
\_| \___|\__,_|\__|\__,_|_|  \___||___/ \_| |_/\_|    \___/  (_)

  ___            _ _
 / _ \          | (_)
/ /_\ \_   _  __| |_  ___
|  _  | | | |/ _` | |/ _ \
| | | | |_| | (_| | | (_) |
\_| |_/\__,_|\__,_|_|\___/

This will featurize folders of audio files if the default_audio_features = ['sox_features']

Get features using SoX library, a workaround by outputting CLI in txt file and
uses a function to extract these features.
'''
import os
import numpy as np

def clean_text(text):
    text=text.lower()
    chars=['a','b','c','d','e','f','g','h','i','j','k','l','m',
           'o','p','q','r','s','t','u','v','w','x','y','z',' ',
           ':', '(',')','-','=',"'.'"]
    for i in range(len(chars)):
        text=text.replace(chars[i],'')

    text=text.split('\n')
    new_text=list()
    # now get new text
    for i in range(len(text)):
        try:
            new_text.append(float(text[i].replace('\n','').replace('n','')))
        except:
            pass
            #print(text[i].replace('\n','').replace('n',''))

    return new_text

def sox_featurize(filename):
    # soxi and stats files
    soxifile=filename[0:-4].replace(' ','_')+'_soxi.txt'
    statfile=filename[0:-4].replace(' ','_')+'_stats.txt'
    if filename.endswith('.mp3'):
    	wavfile= filename[0:-4]+'.wav'
    	os.system('ffmpeg -i %s %s'%(filename,wavfile))
    	os.system('soxi %s > %s'%(wavfile, soxifile))
    	os.system('sox %s -n stat > %s 2>&1'%(wavfile, statfile))
    	os.remove(wavfile)
    else:
    	os.system('soxi %s > %s'%(filename, soxifile))
    	os.system('sox %s -n stat > %s 2>&1'%(filename, statfile))
    # get basic info
    s1=open(soxifile).read()
    s1_labels=['channels','samplerate','precision',
               'filesize','bitrate','sample encoding']
    s1=clean_text(s1)

    s2=open(statfile).read()
    s2_labels=['samples read','length','scaled by','maximum amplitude',
               'minimum amplitude','midline amplitude','mean norm','mean amplitude',
               'rms amplitude','max delta','min delta','mean delta',
               'rms delta','rough freq','vol adj']

    s2=clean_text(s2)

    labels=s1_labels+s2_labels
    features=np.array(s1+s2)
    os.remove(soxifile)
    os.remove(statfile)

    return features,labels

max_stars_repo_name: jim-schwoebel/allie

max_issues_repo_name: jim-schwoebel/allie

max_forks_repo_name: jim-schwoebel/allie


text:
# (c) 2021 Nikolaus Howe

# inspired by https://github.com/openai/gym/blob/master/gym/envs/classic_control/pendulum.py
# and https://github.com/locuslab/mpc.pytorch/blob/07f43da67581b783f4f230ca97b0efbc421773af/mpc/env_dx/pendulum.py

import jax
import jax.numpy as jnp

from typing import Optional

from myriad.systems.base import FiniteHorizonControlSystem
from myriad.custom_types import Control, DState, Params, State, Timestep

# https://github.com/openai/gym/blob/ee5ee3a4a5b9d09219ff4c932a45c4a661778cd7/gym/envs/classic_control/pendulum.py#L101
@jax.jit
def angle_normalize(x):
  return ((x + jnp.pi) % (2 * jnp.pi)) - jnp.pi

class Pendulum(FiniteHorizonControlSystem):
  def __init__(self, g: float = 10., m: float = 1., length: float = 1.):
    # Learnable parameters
    self.g = g
    self.m = m
    self.length = length

    # Fixed parameters
    self.max_speed = 8.
    self.max_torque = 2.
    self.x_0 = jnp.array([0., 0.])
    self.x_T = jnp.array([jnp.pi, 0.])
    self.ctrl_penalty = 0.001

    super().__init__(
      x_0=self.x_0,  # Starting state: position, velocity
      x_T=self.x_T,  # Ending state
      T=15,  # s duration (note, this is not in the original problem)
      bounds=jnp.array([
        [-jnp.pi, jnp.pi],  # theta
        [-self.max_speed, self.max_speed],  # dtheta
        [-self.max_torque, self.max_torque],  # Control bounds
      ]),
      terminal_cost=False,
    )

  def parametrized_dynamics(self, params: Params, x: State, u: Control, t: Optional[Timestep] = None) -> DState:
    u = jnp.clip(u, a_min=-self.max_torque, a_max=self.max_torque)

    g = params['g']
    m = params['m']
    length = params['length']

    theta, dot_theta = x
    theta = angle_normalize(theta)
    dot_theta = jnp.clip(dot_theta, a_min=-self.max_speed, a_max=self.max_speed)
    # print("theta, dot_theta", x)

    dot_dot_theta = (-3. * g / (2. * length) * jnp.sin(theta)
                     + 3. * u / (m * length ** 2)).squeeze() * 0.05

    # print("dot theta", dot_theta)
    # print("dot dot", dot_dot_theta)
    return jnp.array([dot_theta, dot_dot_theta])

  def dynamics(self, x: State, u: Control, t: Optional[Timestep] = None) -> DState:
    u = jnp.clip(u, a_min=-self.max_torque, a_max=self.max_torque)

    theta, dot_theta = x
    theta = angle_normalize(theta)
    dot_theta = jnp.clip(dot_theta, a_min=-self.max_speed, a_max=self.max_speed)
    # print("theta, dot_theta", x)

    dot_dot_theta = (-3. * self.g / (2. * self.length) * jnp.sin(theta)
                     + 3. * u / (self.m * self.length ** 2)).squeeze() * 0.05

    # print("dot theta", dot_theta)
    # print("dot dot", dot_dot_theta)
    return jnp.array([dot_theta, dot_dot_theta])

  def parametrized_cost(self, params: Params, x: State, u: Control, t: Timestep):
    # Do nothing, for now
    return self.cost(x, u, t)

  def cost(self, x: State, u: Control, t: Timestep) -> float:
    # print("state is", x)
    assert len(x) == 2

    theta, dot_theta = x
    return angle_normalize(theta)**2 + 0.1 * dot_theta**2 + self.ctrl_penalty * u**2

if __name__ == "__main__":
  pass

  # import numpy as np
  # import matplotlib.pyplot as plt
  #
  # pd = Pendulum()
  #
  # y1 = np.linspace(-2*jnp.pi, 2*jnp.pi, 20)
  # y2 = np.linspace(-pd.max_speed, pd.max_speed, 20)
  #
  # Y1, Y2 = np.meshgrid(y1, y2)
  #
  # t = 0
  #
  # u, v = np.zeros(Y1.shape), np.zeros(Y2.shape)
  #
  # NI, NJ = Y1.shape
  #
  # for i in range(NI):
  #   for j in range(NJ):
  #     x = Y1[i, j]
  #     y = Y2[i, j]
  #     yprime = pd.dynamics(jnp.array([x, y]), 0, t)
  #     u[i, j] = yprime[0]
  #     v[i, j] = yprime[1]
  #
  # Q = plt.quiver(Y1, Y2, u, v, color='r')
  #
  # plt.xlabel('angle')
  # plt.ylabel('angular velocity')
  # plt.xlim([-2*jnp.pi, 2*jnp.pi])
  # plt.ylim([-pd.max_speed, pd.max_speed])
  # plt.show()
  # def get_frame(self, x, ax=None):
  #   x = util.get_data_maybe(x.view(-1))
  #   assert len(x) == 3
  #   l = self.params[2].item()
  #
  #   cos_th, sin_th, dth = torch.unbind(x)
  #   th = np.arctan2(sin_th, cos_th)
  #   x = sin_th * l
  #   y = cos_th * l
  #
  #   if ax is None:
  #     fig, ax = plt.subplots(figsize=(6, 6))
  #   else:
  #     fig = ax.get_figure()
  #
  #   ax.plot((0, x), (0, y), color='k')
  #   ax.set_xlim((-l * 1.2, l * 1.2))
  #   ax.set_ylim((-l * 1.2, l * 1.2))
  #   return fig, ax

  # def get_true_obj(self):
  #   q = torch.cat((
  #     self.goal_weights,
  #     self.ctrl_penalty * torch.ones(self.n_ctrl)
  #   ))
  #   assert not hasattr(self, 'mpc_lin')
  #   px = -torch.sqrt(self.goal_weights) * self.goal_state  # + self.mpc_lin
  #   p = torch.cat((px, torch.zeros(self.n_ctrl)))
  #   return Variable(q), Variable(p)

reason: import numpy,import jax


text:
from collections import Counter
from shutil import copyfile
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.decomposition import TruncatedSVD
import _pickle as pickle
import re
import path
import os
import numpy as np
import sys
from nltk.corpus import stopwords
import matplotlib.pyplot as plt

train_data=[]
test_data=[]
class_train_flag=[]
class_test_flag=[]
num_files=[]
num_classes = 0
vocabulary = set()
path_test = '../q2data/test'
path_train = '../q2data/train'

def read_freq_file(path):
	with open(path, errors='ignore') as f1:
		shop = f1.read()
		regex = r'\b\w+\b'
		list1=re.findall(regex,shop)
		list1 = [x for x in list1 if not any(c.isdigit() for c in x)]
		c=Counter(x.strip() for x in list1)
		return c
def create_test_data():
	global num_classes
	global num_files
	global train_data
	global test_data
	global class_train_flag
	global class_test_flag

	num_classes = len([f for f in os.listdir(os.path.join(path_train)) ])
	num_files = [0]*num_classes
	for class_no in range(0,num_classes):
		for f in os.listdir(os.path.join(path_test,str(class_no).zfill(1))):
			# print(f)
			os.remove(os.path.join(path_test,str(class_no).zfill(1), f))

		num_files[class_no] = len([f for f in os.listdir(os.path.join(path_train,str(class_no).zfill(1))) \
				if os.path.isfile(os.path.join(path_train,str(class_no).zfill(1), f))])

		for file_no in range(int(num_files[class_no]*0.8)+1,num_files[class_no]+1):
			copyfile(os.path.join(path_train,str(class_no).zfill(1),str(file_no).zfill(3)+'.txt'),\
			os.path.join(path_test,str(class_no).zfill(1),str(file_no).zfill(3)+'.txt'))

		for f in os.listdir(os.path.join(path_test,str(class_no).zfill(1))):
			test_data.append(read_freq_file(os.path.join(path_test,str(class_no).zfill(1),f)))
			class_test_flag.append(class_no)

		for file_no in range(1,int(num_files[class_no]*0.8)+1):
			train_data.append(read_freq_file(os.path.join(path_train,str(class_no).zfill(1),str(file_no).zfill(3)+'.txt')))
			class_train_flag.append(class_no)
def build_lexicon(train_data):
	lexicon = set()
	for doc in train_data:
		lexicon.update([x for x in doc])
	filtered_words = [word for word in lexicon if word not in stopwords.words('english')]

	return filtered_words
def tf(term, document):
	if term in document:
		return document[term]
	else:
		return 0
def find_dim(threshold, Vt , s):

	num_dim=0
	# pc = U*np.diag(s)
	# pc = pc[:,::-1]
	# explained_variance = np.var(pc, axis=0)
	# full_variance = np.var(tf_idf_matrix,axis=0)
	# explained_variance_ratio = explained_variance / full_variance.sum()
	# print(explained_variance_ratio.cumsum())
	cumsum = (s/s.sum()).cumsum()
	# cumsum = explained_variance_ratio.cumsum()
	# print(cumsum)

	# print(threshold)
	for i in range(0,len(s)):
		num_dim+=1
		if cumsum[i]>threshold:
			break
	# num_dim = cumsum[np.where(cumsum>threshold)].shape[1]
	print('Number of Dimensions for threshold-> ',threshold, '\t',num_dim)
	return Vt[0:num_dim,:].T

	# B = U[:,0:num_dim]*((np.diag(s))[0:num_dim,0:num_dim])
	# B_test = test_tfidf*V[0:num_dim,:].T

	# print (s)
	# print (V)
def sign(num):
	if np.asscalar(num) >0:
		return 1
	else:
		return -1
def OVAPerceptronTraining(x_train, y_train):
	num_obs = len(x_train)
	a,num_feat = x_train[0].shape
	output=[]
	for i in range(0,num_classes):
		output.append(np.zeros(num_feat))
	c1=0
	c2=0
	for i in range(0,num_obs):
		actual = y_train[i]
		pred = 0
		prod_max = np.asscalar(np.dot(x_train[i],output[0].T))

		for class_no in range(0,num_classes):
			prod = np.asscalar(np.dot(x_train[i],output[class_no].T))

			if prod > prod_max:
				pred = class_no
				prod_max = prod

		if pred != actual:
			output[actual] = output[actual]+x_train[i]
			output[pred] = output[pred]-x_train[i]
			c1+=1
		else:
			c2+=1
	print(c1,c2)
	return output
def OVAPerceptronTesting(output, x_test, y_test):
	correct_v=0
	incorrect_v=0
	for ind in range(0,len(x_test)):
		pred = 0
		actual = y_test[ind]
		prod_max=0
		w = output[0]

		prod_max=np.asscalar(np.dot(x_test[ind],w.T))

		for class_number in range(0,num_classes):
			w = output[class_number]
			prod=np.asscalar(np.dot(x_test[ind],w.T))

			if prod > prod_max:
				pred = class_number
				prod_max = prod

		if pred == actual:
			correct_v+=1
		else:
			incorrect_v+=1
	print(correct_v, incorrect_v)
	return float(float(correct_v)/float(incorrect_v+correct_v))
def cosine_similarity( x_train, y_train, x_test, y_test):
	correct_v=0
	incorrect_v=0
	for test_doc in range(0,len(x_test)):
		cosines=[]
		for train_doc in range(0,len(x_train)):
			a=x_train[train_doc]
			b=x_test[test_doc]
			moda = np.linalg.norm(a)
			modb = np.linalg.norm(b)
			cosines.append((y_train[train_doc], np.dot(a,b.T)/moda/modb))
		top_10 = sorted(cosines, key=lambda x: x[1], reverse = True)[0:10]
		# print (top_10)
		count=[0]*num_classes
		for i in top_10:
			count[i[0]]+=1
		pred = count.index(max(count))
		if pred == y_test[test_doc]:
			correct_v+=1
		else:
			incorrect_v+=1
	return float(float(correct_v)/float(incorrect_v+correct_v))
def perform_tfidf(train_data, test_data, vocabulary):

	doc_term_matrix = []
	for doc in train_data+test_data:
	    tf_vector = [tf(word, doc) for word in vocabulary]
	    doc_term_matrix.append(tf_vector)

	doc_term_matrix = np.array(doc_term_matrix)

	tfidf = TfidfTransformer(norm="l2")
	tfidf.fit(doc_term_matrix)
	tf_idf_matrix = np.matrix(tfidf.transform(doc_term_matrix).toarray())

	tf_idf_matrix-=np.matrix((np.mean(tf_idf_matrix, 0)))

	U, s, V = np.linalg.svd( tf_idf_matrix , full_matrices=False)

	return s, V
	# return s,V [0:num_dim,:].T
def perform_tfidf_validation(train_data, test_data, vocabulary):
	doc_term_matrix = []
	for doc in train_data:
	    tf_vector = [tf(word, doc) for word in vocabulary]
	    doc_term_matrix.append(tf_vector)

	doc_term_matrix = np.array(doc_term_matrix)
	tfidf = TfidfTransformer(norm="l2")
	tfidf.fit(doc_term_matrix)
	doc_term_matrix = doc_term_matrix[0:len(train_data)]
	tf_idf_matrix = np.matrix(tfidf.transform(doc_term_matrix).toarray())
	tf_idf_matrix-=np.matrix((np.mean(tf_idf_matrix, 0)))
	return tf_idf_matrix, tfidf
def perform_transform(test_data, tfidf, vocabulary):

	doc_term_matrix = []
	for doc in test_data:
	    tf_vector = [tf(word, doc) for word in vocabulary]
	    doc_term_matrix.append(tf_vector)

	doc_term_matrix = np.array(doc_term_matrix)
	tf_idf_matrix = np.matrix(tfidf.transform(doc_term_matrix).toarray())
	tf_idf_matrix-=np.matrix((np.mean(tf_idf_matrix, 0)))
	return tf_idf_matrix
def dimension_reduction(V, Matrix):
	return Matrix*V
def perform_tfidf_query(train_data):
	vocabulary = build_lexicon(train_data)
	doc_term_matrix = []
	for doc in train_data:
	    tf_vector = [tf(word, doc) for word in vocabulary]
	    doc_term_matrix.append(tf_vector)

	doc_term_matrix = np.array(doc_term_matrix)
	tfidf = TfidfTransformer(norm="l2")
	tfidf.fit(doc_term_matrix)
	tf_idf_matrix = np.matrix(tfidf.transform(doc_term_matrix).toarray())
	tf_idf_matrix-=np.matrix((np.mean(tf_idf_matrix, 0)))
	U, s, V = np.linalg.svd( tf_idf_matrix , full_matrices=False)
	return tf_idf_matrix, tfidf, vocabulary, V, s
	# return s,V [0:num_dim,:].T
def cosine_similarity_query(x_train, y_train, x_test):
	for test_doc in range(0,len(x_test)):
		cosines=[]
		for train_doc in range(0,len(x_train)):
			a=x_train[train_doc]
			b=x_test[test_doc]
			moda = np.linalg.norm(a)
			modb = np.linalg.norm(b)
			cosines.append((y_train[train_doc], np.dot(a,b.T)/moda/modb))
		top_10 = sorted(cosines, key=lambda x: x[1], reverse = True)[0:10]
		count=[0]*num_classes
		for i in top_10:
			count[i[0]]+=1
		pred = count.index(max(count))
		return pred
	return 0
def create_test_data_query():
	global num_classes
	global num_files
	global train_data
	global test_data
	global class_train_flag
	global class_test_flag
	num_classes = len([f for f in os.listdir(os.path.join(path_train)) ])
	num_files = [0]*num_classes
	for class_no in range(0,num_classes):
		num_files[class_no] = len([f for f in os.listdir(os.path.join(path_train,str(class_no).zfill(1))) \
				if os.path.isfile(os.path.join(path_train,str(class_no).zfill(1), f))])
		for file_no in range(1,int(num_files[class_no])+1):
			train_data.append(read_freq_file(os.path.join(path_train,str(class_no).zfill(1),str(file_no).zfill(3)+'.txt')))
			class_train_flag.append(class_no)
# if __name__ == "__main__":

if sys.argv[1]=='1':
	if len(sys.argv)>=3:
		path_train = sys.argv[2]

	if len(sys.argv)>=4:
		path_test = sys.argv[3]
	create_test_data()
	vocabulary = build_lexicon(train_data)
	s, Vt = perform_tfidf(train_data, test_data, vocabulary)
	B, tfidf = perform_tfidf_validation(train_data, test_data, vocabulary)
	B_test = perform_transform(test_data, tfidf, vocabulary)
	with open("Save_SVD.data",'wb') as fp:
	    pickle.dump(s,fp)
	    pickle.dump(Vt,fp)
	    pickle.dump(B,fp)
	    pickle.dump(B_test,fp)
elif sys.argv[1]=='2':
	print("Threshold\tNum Dim")
	with open("Save_SVD.data",'rb') as fp:
	    s=pickle.load(fp)
	    Vt=pickle.load(fp)
	for threshold in range(50,110,5):
		find_dim(float(threshold/100),Vt , s)
elif sys.argv[1]=='3':
	if len(sys.argv)>=3:
		path_train = sys.argv[2]

	if len(sys.argv)>=4:
		path_test = sys.argv[3]
	create_test_data()
	vocabulary = build_lexicon(train_data)
	with open("Save_SVD.data",'rb') as fp:
	    s=pickle.load(fp)
	    Vt=pickle.load(fp)
	    B=pickle.load(fp)
	    B_test=pickle.load(fp)

	X_plot=[]
	Y_plotp=[]
	Y_plotc=[]
	print('threshold\taccuracy_perceptron\taccuracy_cosine')
	for threshold in range(60,101,5):
		V = find_dim(float(threshold/100),Vt , s)

		B_transformed = dimension_reduction(V, B)
		B_test_transformed = dimension_reduction(V, B_test)
		x_train=[]
		y_train=[]
		for ind in range(0,len(B_transformed)):
			# x_train.append(np.append(np.squeeze(B_transformed[ind]),np.ones((1,1)), axis=1))
			x_train.append(np.squeeze(B_transformed[ind]))
			y_train.append(class_train_flag[ind])

		model = OVAPerceptronTraining(x_train, y_train)

		x_test=[]
		y_test=[]
		for ind in range(0,len(B_test_transformed)):
			# x_test.append(np.append(np.squeeze(B_test_transformed[ind]),np.ones((1,1)), axis=1))
			x_test.append(np.squeeze(B_test_transformed[ind]))
			# x_test.append(np.squeeze(B_test_transformed[ind]))
			y_test.append(class_test_flag[ind])

		accuracy_perceptron = OVAPerceptronTesting(model, x_test, y_test)
		accuracy_cosine = cosine_similarity(x_train, y_train, x_test, y_test)
		print(threshold,'\t', accuracy_perceptron,'\t', accuracy_cosine)

		X_plot.append(threshold)
		Y_plotp.append(accuracy_perceptron*100)
		Y_plotc.append(accuracy_cosine*100)
	plt.plot(X_plot, Y_plotp, color = 'blue', label='Perceptron Accuracy')
	plt.plot(X_plot, Y_plotc, color = 'black', label='Cosine Sim Accuracy')

	for i in range(0,len(X_plot)):
		plt.plot(X_plot[i], Y_plotp[i], 'ro')

	for i in range(0,len(X_plot)):
		plt.plot(X_plot[i], Y_plotc[i], 'ro')

	plt.axis([50, 110, 60, 110])
	plt.legend(loc='best')
	plt.xlabel('Threshold (in %)', fontsize=18)
	plt.ylabel('Accuracy', fontsize=18)
	plt.show()
elif sys.argv[1]=='4':
	if len(sys.argv)>=3:
		path_train = sys.argv[2]

	if len(sys.argv)>=4:
		path_test = sys.argv[3]

	if len(sys.argv)>=5:
		class_already = int(sys.argv[4])

	create_test_data_query()
	query_file = read_freq_file(path_test)
	tfidf_matrix, tfidf, vocabulary, Vt, s = perform_tfidf_query(train_data)
	query_tfidf = perform_transform([query_file], tfidf, vocabulary)

	V = find_dim(float(95/100),Vt , s)

	train_data = dimension_reduction(V,tfidf_matrix)
	query_tfidf = dimension_reduction(V,query_tfidf)

	predicted_class = cosine_similarity_query(train_data, class_train_flag, query_tfidf)
	print("Predicted Class-> ",predicted_class,'\n',"Query Class-> ", class_already)


text:
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import unittest
import numpy as np
import tensorflow as tf
import onnx
from onnx_tf.backend import run_node, prepare
from onnx import helper
from onnx.onnx_pb2 import TensorProto

class TestModel(unittest.TestCase):
  """ Tests for models
  """
  def test_relu_node_inplace(self):
    X = np.random.randn(3, 2).astype(np.float32)
    Y_ref = np.clip(X, 0, np.inf)

    node_def = helper.make_node(
      "Relu", ["X"], ["X"])

    graph_def = helper.make_graph(
      [node_def],
      name="test",
      inputs=[helper.make_tensor_value_info("X", TensorProto.FLOAT, [3, 2])],
      outputs=[helper.make_tensor_value_info("X", TensorProto.FLOAT, [3, 2])])
    tf_rep = prepare(helper.make_model(graph_def))
    output = tf_rep.run({"X": X})
    np.testing.assert_almost_equal(output.X, Y_ref)

  def test_initializer(self):
    X = np.array([[1, 2], [3, 4]]).astype(np.float32)
    Y = np.array([[1, 2], [3, 4]]).astype(np.float32)
    weight = np.array([[1, 0], [0, 1]])
    graph_def = helper.make_graph(
        [helper.make_node("Add", ["X", "Y"], ["Z0"]),
         helper.make_node("Cast", ["Z0"], ["Z"], to="float"),
         helper.make_node("Mul", ["Z", "weight"], ["W"]),
         helper.make_node("Tanh", ["W"], ["W"]),
         helper.make_node("Sigmoid", ["W"], ["W"])],
        name="test_initializer",
        inputs=[
            helper.make_tensor_value_info("X", TensorProto.FLOAT, (2, 2)),
            helper.make_tensor_value_info("Y", TensorProto.FLOAT, (2, 2)),
            helper.make_tensor_value_info("weight", TensorProto.FLOAT, (2, 2)),
        ],
        outputs=[
            helper.make_tensor_value_info("W", TensorProto.FLOAT, (2, 2))
        ],
        initializer=[helper.make_tensor("weight",
                                 TensorProto.FLOAT,
                                 [2, 2],
                                 weight.flatten().astype(float))]
    )

    def sigmoid(x):
        return 1 / (1 + np.exp(-x))

    W_ref = sigmoid(np.tanh((X + Y) * weight))
    tf_rep = prepare(helper.make_model(graph_def))
    output = tf_rep.run({"X": X, "Y": Y})
    np.testing.assert_almost_equal(output["W"], W_ref)

if __name__ == '__main__':
  unittest.main()

max_stars_repo_name: tjingrant/onnx-tf

max_issues_repo_name: tjingrant/onnx-tf

max_forks_repo_name: tjingrant/onnx-tf


text:
"""
Tests that raw data inputs function correctly.
"""

import sys; sys.path.insert(0, '../')
import geoplot as gplt
import unittest
import geopandas as gpd
import matplotlib.pyplot as plt
from shapely.geometry import Point, LineString
import numpy as np
import pandas as pd

# Point-type DataFrame input.
list_gaussian_points = gplt.utils.gaussian_points(n=4)
series_gaussian_points = gpd.GeoSeries(list_gaussian_points)
dataframe_gaussian_points = gpd.GeoDataFrame(
    geometry=series_gaussian_points
).assign(hue_var=[1,2,3,4])

# Polygon-type DataFrame input.
list_gaussian_polys = gplt.utils.gaussian_polygons(
    gplt.utils.gaussian_points(n=1000), n=2
).append(
    gplt.utils.gaussian_multi_polygons(gplt.utils.gaussian_points(n=1000), n=2)
)
series_gaussian_polys = gpd.GeoSeries(list_gaussian_polys)
dataframe_gaussian_polys = gpd.GeoDataFrame(
    geometry=series_gaussian_polys
).assign(hue_var=[1,2,3,4])

# Hue input.
list_hue_values = [1, 2, 3, 4]
series_hue_values = pd.Series(list_hue_values)
def map_hue_values(): return map(lambda i: list_hue_values[i], list(range(len(list_hue_values))))

# (Sankey) point input.
# Start and End variables.
list_start_points = [Point(a + 2, a - 2) for a in range(0, 4)]
list_end_points = [Point(a - 2, a + 2) for a in range(1, 5)]
series_start_points = gpd.GeoSeries(list_start_points)
series_end_points = gpd.GeoSeries(list_end_points)
def map_start_points():
    return map(lambda i: list_start_points[i], list(range(len(list_start_points))))
def map_end_points():
    return map(lambda i: list_end_points[i], list(range(len(list_end_points))))
dataframe_gaussian_points = dataframe_gaussian_points.assign(
    starts=list_start_points, ends=list_end_points
)

# (Sankey) paths.
list_paths = [
    LineString([[a.x, a.y], [b.x, b.y]]) for a, b in zip(list_start_points, list_end_points)
]
series_paths = gpd.GeoSeries(list_paths)
def map_paths(): return map(lambda i: list_paths[i], list(range(len(list_paths))))
dataframe_gaussian_points = dataframe_gaussian_points.assign(paths=list_paths)

# (Aggplot) geometry.
dataframe_gaussian_points = dataframe_gaussian_points.assign(
    mock_category=np.random.randint(1, 5)
)
aggplot_geometries = dataframe_gaussian_polys.set_index('hue_var', drop=True)

class TestDataInputFormats(unittest.TestCase):

    def test_pointplot(self):
        try:
            gplt.pointplot(series_gaussian_points, k=2)
            gplt.pointplot(dataframe_gaussian_points, k=2)

            gplt.pointplot(dataframe_gaussian_points, hue=list_hue_values, k=None)
            gplt.pointplot(dataframe_gaussian_points, hue=series_hue_values, k=None)
            gplt.pointplot(dataframe_gaussian_points, hue=map_hue_values(), k=None)
            gplt.pointplot(dataframe_gaussian_points, hue='hue_var', k=None)
        finally:
            plt.close('all')

    def test_kdeplot(self):
        try:
            gplt.kdeplot(series_gaussian_points)
            gplt.kdeplot(dataframe_gaussian_points)

            gplt.kdeplot(dataframe_gaussian_points, hue=list_hue_values)
            gplt.kdeplot(dataframe_gaussian_points, hue=series_hue_values)
            gplt.kdeplot(dataframe_gaussian_points, hue=map_hue_values)
            gplt.kdeplot(dataframe_gaussian_points, hue='hue_var')
        finally:
            plt.close('all')

    def test_cartogram(self):
        try:
            gplt.cartogram(series_gaussian_polys, scale=list_hue_values)
            gplt.cartogram(dataframe_gaussian_polys, scale=list_hue_values)

            gplt.cartogram(dataframe_gaussian_polys, hue=list_hue_values, scale=list_hue_values)
            gplt.cartogram(dataframe_gaussian_polys, hue=series_hue_values, scale=list_hue_values)
            gplt.cartogram(dataframe_gaussian_polys, hue=map_hue_values(), scale=list_hue_values)
            gplt.cartogram(dataframe_gaussian_polys, hue='hue_var', scale=list_hue_values)
        finally:
            plt.close('all')

    def test_polyplot(self):
        try:
            gplt.polyplot(series_gaussian_polys)
            gplt.polyplot(dataframe_gaussian_polys)
        finally:
            plt.close('all')

    def test_choropleth(self):
        try:
            gplt.choropleth(series_gaussian_polys, hue=list_hue_values)
            gplt.choropleth(dataframe_gaussian_polys, hue=list_hue_values)

            gplt.choropleth(dataframe_gaussian_polys, hue=list_hue_values)
            gplt.choropleth(dataframe_gaussian_polys, hue=series_hue_values)
            gplt.choropleth(dataframe_gaussian_polys, hue=map_hue_values())
            gplt.choropleth(dataframe_gaussian_polys, hue='hue_var')
        finally:
            plt.close('all')

    def test_sankey(self):
        try:
            gplt.sankey(start=map_start_points(), end=map_end_points())
            gplt.sankey(start=map_start_points(), end=map_end_points())

            gplt.sankey(start=list_start_points, end=list_end_points)
            gplt.sankey(start=list_start_points, end=list_end_points)

            gplt.sankey(start=series_start_points, end=series_end_points)
            gplt.sankey(start=series_start_points, end=series_end_points)

            gplt.sankey(start=map_start_points(), end=map_end_points())
            gplt.sankey(start=map_start_points(), end=map_end_points())

            gplt.sankey(dataframe_gaussian_points, start='starts', end='ends')

            gplt.sankey(path=list_paths)
            gplt.sankey(path=series_paths)
            gplt.sankey(path=map_paths())

            gplt.sankey(dataframe_gaussian_points, path='paths')

        finally:
            plt.close('all')

    def test_aggplot(self):
        try:
            gplt.aggplot(series_gaussian_points, hue=list_hue_values)
            gplt.aggplot(dataframe_gaussian_points, hue=list_hue_values)

            gplt.aggplot(dataframe_gaussian_points, hue=list_hue_values)
            gplt.aggplot(dataframe_gaussian_points, hue=series_hue_values)
            gplt.aggplot(dataframe_gaussian_points, hue=map_hue_values())
            gplt.aggplot(dataframe_gaussian_points, hue='hue_var')

            gplt.aggplot(dataframe_gaussian_points, hue=list_hue_values, by='mock_category')
            # series
            gplt.aggplot(dataframe_gaussian_points, hue=list_hue_values,
                         by=dataframe_gaussian_points['mock_category'])
            # list
            gplt.aggplot(dataframe_gaussian_points, hue=list_hue_values,
                         by=list(dataframe_gaussian_points['mock_category']))
            # map
            gplt.aggplot(dataframe_gaussian_points, hue=list_hue_values,
                         by=map(lambda v: v, list(dataframe_gaussian_points['mock_category'])))
            gplt.aggplot(dataframe_gaussian_points, hue=list_hue_values, by='mock_category',
                         geometry=aggplot_geometries)
            gplt.aggplot(dataframe_gaussian_points, hue=list_hue_values,
                         by=dataframe_gaussian_points['mock_category'],
                         geometry=aggplot_geometries)  # Series
            gplt.aggplot(dataframe_gaussian_points, hue=list_hue_values,
                         by=list(dataframe_gaussian_points['mock_category']),
                         geometry=aggplot_geometries)  # List
            gplt.aggplot(dataframe_gaussian_points, hue=list_hue_values,
                         by=map(lambda v: v, list(dataframe_gaussian_points['mock_category'])),
                         geometry=aggplot_geometries)  # Map

        finally:
            plt.close('all')

max_stars_repo_name: redfrexx/geoplot

max_issues_repo_name: redfrexx/geoplot

max_forks_repo_name: redfrexx/geoplot


text:
import math
import numpy as np
from sklearn import preprocessing, model_selection, svm
from sklearn.linear_model import LinearRegression
import matplotlib.pyplot as plt
from matplotlib import style
import pandas as pd
import datetime
import tensorflow as tf

mnist = tf.keras.datasets.mnist

(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train = x_train[:, :, :, np.newaxis]
x_test = x_test[:, :, :, np.newaxis]

n_classes = 10
y_train = tf.keras.utils.to_categorical(y_train, n_classes)
y_test = tf.keras.utils.to_categorical(y_test, n_classes)

x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
x_train /= 255
x_test /= 255

inputs = tf.keras.Input(shape=(28, 28, 1), name='data')
x = tf.keras.layers.Conv2D(6, kernel_size=(5, 5), strides=(1, 1), activation='relu', padding='same')(inputs)
x = tf.keras.layers.MaxPooling2D(2,strides=(2,2))(x)
x = tf.keras.layers.Conv2D(16, kernel_size=(5, 5), strides=(1, 1), activation='relu', padding='valid')(x)
x = tf.keras.layers.MaxPooling2D(2,strides=(2,2))(x)
x = tf.keras.layers.Flatten()(x)
x = tf.keras.layers.Dense(120, activation='relu')(x)
x = tf.keras.layers.Dense(84, activation='relu')(x)
outputs = tf.keras.layers.Dense(n_classes, activation='softmax')(x)

model = tf.keras.Model(inputs=inputs, outputs=outputs, name='lenet')

model.summary()

model.compile(
        optimizer=tf.keras.optimizers.SGD(learning_rate=0.01),
        loss=tf.keras.losses.categorical_crossentropy,
        metrics=['accuracy']
    )

tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir="fit_logs\\", histogram_freq=1)

model.fit(x_train, y_train, batch_size=100, epochs=20, validation_data=(x_test, y_test), callbacks=[tensorboard_callback])

max_stars_repo_name: riciche/SimpleCVReproduction

max_issues_repo_name: riciche/SimpleCVReproduction

max_forks_repo_name: riciche/SimpleCVReproduction


text:
import itertools
import math
import numpy as np
import pandas as pd

def words_of_length(alphabet_size, word_length):
    return tuple(itertools.product(range(1, alphabet_size+1), repeat=word_length))

def words(alphabet_size, max_word_length=None):
    if max_word_length is None: max_word_length = alphabet_size
    return sum((words_of_length(alphabet_size, i) for i in range(1, max_word_length+1)), ())

def signature_element(df, word):
    if len(word) == 0:
        return 1.
    elif len(df) == 2:
        result = 1./math.factorial(len(word))
        for letter in word:
            result *= df.iloc[1,letter-1] - df.iloc[0,letter-1]
        return result
    else:
        result = 0.
        for i in range(0, len(word)+1):
            result += signature_element(df[:2], word[0:i]) * signature_element(df[1:], word[i:])
        return result

def signature(df, max_word_length=None):
    return {w: signature_element(df, w) for w in words(len(df.columns), max_word_length)}

max_stars_repo_name: saarahrasheed/tsa

max_issues_repo_name: saarahrasheed/tsa

max_forks_repo_name: saarahrasheed/tsa


text:
import numpy as np

from core import rolling_func, offset, divide, replace_nan

def rolling_change(price, window=1):
    return offset(price[window:] / price[:-window], window)

def rolling_diff(series, window=1):
    return offset(series[window:] / series[:-window], window)

def rolling_max(series, window=20):
    return rolling_func(series, window, [np.max])[0]

def rolling_min(series, window=20):
    return rolling_func(series, window, [np.min])[0]

def true_price(high, low, close):
    return (high + low + close) / 3

def aroon(high, low, window=20):
    aroon_up = rolling_func(high, window + 1, [np.argmax])[0] / window
    aroon_down = rolling_func(low, window + 1, [np.argmin])[0] / window
    return aroon_down * 100, aroon_up * 100

def aroon_oscillator(high, low, window=20):
    aroon_down, aroon_up = aroon(high, low, window=window)
    return aroon_up - aroon_down

def b_bands(price, window=20, multiplier=2):
    mean, std = rolling_func(price, window, [np.mean, np.std])
    lower = mean - multiplier * std
    upper = mean + multiplier * std
    return upper, mean, lower

def b_bands_percent(price, window=20, multiplier=2):
    upper, _, lower = b_bands(price, window=window, multiplier=multiplier)
    out = divide(price - lower, upper - lower)
    return replace_nan(out)

def vortex_index(high, low, close, window=20):
    true_range = np.maximum.reduce(
        [high[1:] - low[1:], high[1:] - close[:-1], low[1:] - close[:-1]])
    true_range = offset(rolling_func(true_range, window, [np.sum])[0], 1)
    uptrend = offset(np.abs(high[1:] - low[:-1]), 1)
    downtrend = offset(np.abs(low[1:] - high[:-1]), 1)

    vi_plus = divide(rolling_func(uptrend, window, [np.sum])[0], true_range)
    vi_minus = divide(rolling_func(downtrend, window, [np.sum])[0], true_range)
    vi_plus[:window] = np.nan
    vi_minus[:window] = np.nan
    return replace_nan(vi_plus), replace_nan(vi_minus)

def roc(series, widnow=20):
    return offset(
        rolling_diff(series, window)[widnow:] / series[:-widnow], window)

max_stars_repo_name: JoarGruneau/yatal

max_issues_repo_name: JoarGruneau/yatal

max_forks_repo_name: JoarGruneau/yatal


text:
import numpy as np

class FileReader:

    @staticmethod
    def file_reader(file_path='roms/Tetris.ch8'):
        opened_file = open(file_path, "rb").read()
        return opened_file

    @staticmethod
    def load_binary_to_buffer(binary_file):
        buffer = np.zeros(len(binary_file), dtype=np.uint8)
        for index, value in enumerate(binary_file):
            buffer[index] = value
        return buffer


text:
import theano
import theano.tensor as T
import numpy as np
from theano_utils import floatX
from rng import t_rng

class SGD():

    def __init__(self, lr=0.01, alpha=0.9, decay=True):
        self.__dict__.update(locals())

    def __call__(self, params, grads):
        updates = []

        t_prev = theano.shared(floatX(0.))

        t = t_prev + 1
        for p,g in zip(params, grads):
            value = p.get_value(borrow=True)
            velocity = theano.shared(np.zeros(value.shape, dtype=value.dtype), broadcastable=p.broadcastable)
            if self.decay:
                curr_lr = self.lr * T.cast((1 + t)**(-.55), theano.config.floatX)
            else:
                curr_lr = self.lr
            step = self.alpha * velocity + curr_lr * g

            updated_p = p - step
            updates.append((p, updated_p))
            updates.append((t_prev, t))

        return updates

class Adam():

    def __init__(self, lr=0.001, b1=0.9, b2=0.999, e=1e-8, l=1-1e-8):
        self.__dict__.update(locals())

    def __call__(self, params, grads):
        updates = []

        t_prev = theano.shared(floatX(0.))

        # Using theano constant to prevent upcasting of float32
        one = T.constant(1)

        t = t_prev + 1
        a_t = self.lr*T.sqrt(one-self.b2**t)/(one-self.b1**t)

        for param, g_t in zip(params, grads):
            value = param.get_value(borrow=True)
            m_prev = theano.shared(np.zeros(value.shape, dtype=value.dtype), broadcastable=param.broadcastable)
            v_prev = theano.shared(np.zeros(value.shape, dtype=value.dtype), broadcastable=param.broadcastable)
            m_t = self.b1*m_prev + (one-self.b1)*g_t
            v_t = self.b2*v_prev + (one-self.b2)*g_t**2
            step = a_t*m_t/(T.sqrt(v_t) + self.e)

            updates.append((m_prev, m_t))
            updates.append((v_prev, v_t))
            updates.append((param, param-step))

        updates.append((t_prev, t))
        return updates

class Adagrad():

    def __init__(self, lr=0.01, alpha=-1, epsilon=1e-6):
        self.__dict__.update(locals())

    def __call__(self, params, grads):
        updates = []

        for p,g in zip(params,grads):
            acc = theano.shared(p.get_value() * 0.)
            if self.alpha > 0:
                acc_t = self.alpha * acc + (1. - self.alpha) * (g ** 2)
            else:
                acc_t = acc + g ** 2
            updates.append((acc, acc_t))

            p_t = p - (self.lr / T.sqrt(acc_t + self.epsilon)) * g
            updates.append((p, p_t))
        return updates

reason: import numpy,import theano,from theano