text:
#!/usr/bin/env python
"""Provides layout in 2D of vector instances."""

from collections import defaultdict
import networkx as nx
import matplotlib
import matplotlib.pyplot as plt
from scipy.spatial import ConvexHull
from matplotlib.collections import PatchCollection
from matplotlib.patches import Polygon
import numpy as np
from sklearn.preprocessing import OneHotEncoder
from sklearn.metrics.pairwise import pairwise_kernels

import logging
logger = logging.getLogger(__name__)

class GraphEmbedder(object):
    """Transform high dimensional labeled vectors to two dimensional vectors.

    A discrete label for each instance is expected.
    A graph is built where nodes are instances and there exist two types
    of edges: the 'knn' edges and the 'k_shift' edges.
    A knn edge is an edge to the k-th nearest instance that has the same
    label.
    A k_shift edge is an edge to the k-th nearest instance that is denser
    and has a different label.
    The density is defined as the sum of the pairwise cosine similarity between
    an instance and all the other instances.
    The desired edge length is the euclidean distance between the instances.
    If the endpoints of an edge have the same label then the desired distance
    is divided by 1 + class_bias.
    A k-shift edge is deleted if at least one of the endpoints of is an
    outlier.
    Outlier nodes are defined as those instances that have no mutual
    k=knn_outlier neighbors.

    Finally the embedding is computed as the 2D coordinates of the
    corresponding graph embedding using the force layout algorithm from
    Tomihisa Kamada, and Satoru Kawai. "An algorithm for drawing general
    undirected graphs.", Information processing letters 31, no. 1 (1989): 7-15.
    """

    def __init__(self,
                 k=5,
                 class_bias=2,
                 random_state=1,
                 metric='cosine', **kwds):
        """Constructor."""
        self.name = self.__class__.__name__
        self.__version__ = '1.0.1'

        self.random_state = random_state
        self.k = k
        self.class_bias = class_bias
        self.metric = metric
        self.kwds = kwds

        self.probs = None
        self.target_pred = None
        self.graph = None
        self.knn_graph = None

        logger.debug('%s' % str(self))

    def __str__(self):
        """String."""
        def serialize_dict(the_dict, full=True, offset='small'):
            """serialize_dict."""
            if the_dict:
                text = []
                for key in sorted(the_dict):
                    if offset == 'small':
                        line = '%10s: %s' % (key, the_dict[key])
                    elif offset == 'large':
                        line = '%25s: %s' % (key, the_dict[key])
                    elif offset == 'very_large':
                        line = '%50s: %s' % (key, the_dict[key])
                    else:
                        raise Exception('unrecognized option: %s' % offset)
                    line = line.replace('\n', ' ')
                    if full is False:
                        if len(line) > 100:
                            line = line[:100] + '  ...  ' + line[-20:]
                    text.append(line)
                return '\n'.join(text)
            else:
                return ""

        return "%s:\n%s" % (self.__class__, serialize_dict(self.__dict__))

    def transform(self, data_matrix=None, target=None):
        """transform."""
        if target is None:
            target = np.array([1] * data_matrix.shape[0])
        instance_graph = self.build_graph(data_matrix=data_matrix,
                                          target=target,
                                          k=self.k)
        self.annotate_graph_with_graphviz_layout(instance_graph)
        embedded_data_matrix = [instance_graph.node[v]['pos']
                                for v in instance_graph.nodes()]
        self.instance_graph = instance_graph

        self.embedded_data_matrix = np.array(embedded_data_matrix)
        self.target = target
        return self.embedded_data_matrix

    def build_graph(self, data_matrix=None, target=None, k=3):
        """Build graph."""
        size = data_matrix.shape[0]
        # make kernel
        kernel_matrix = pairwise_kernels(data_matrix,
                                         metric=self.metric,
                                         **self.kwds)
        # compute instance density as average pairwise similarity
        density = np.sum(kernel_matrix, 0) / size
        # compute list of nearest neighbors
        kernel_matrix_sorted_ids = np.argsort(-kernel_matrix)
        # make matrix of densities ordered by nearest neighbor
        density_matrix = density[kernel_matrix_sorted_ids]

        # make a graph with instances as nodes
        graph = nx.Graph()
        for v in range(size):
            graph.add_node(v, group=target[v], outlier=False)

        # build knn edges
        if k > 0:
            # find the closest selected instance and instantiate knn edges
            graph = self._add_knn_links(
                graph,
                target,
                kernel_matrix=kernel_matrix,
                kernel_matrix_sorted_ids=kernel_matrix_sorted_ids,
                nneighbors_th=k)
            self._annotate_outliers(
                graph,
                nneighbors_th=k,
                kernel_matrix=kernel_matrix,
                kernel_matrix_sorted_ids=kernel_matrix_sorted_ids)

        # build shift tree
        for th in range(1, k + 1):
            link_ids = self._kernel_shift_links(
                kernel_matrix=kernel_matrix,
                density_matrix=density_matrix,
                kernel_matrix_sorted_ids=kernel_matrix_sorted_ids,
                k_quick_shift=th,
                target=target)
            for i, link in enumerate(link_ids):
                if i != link:
                    graph.add_edge(i, link, edge_type='shift', rank=th)
        graph = self._compute_edge_len(graph, data_matrix, target)
        return graph

    def _diameter(self, data_matrix):
        curr_point = data_matrix[0]
        for itera in range(3):
            # find furthest point from curr_point
            id = np.argmax(np.array([np.linalg.norm(point - curr_point)
                                     for point in data_matrix]))
            curr_point = data_matrix[id]
        return max([np.linalg.norm(point - curr_point)
                    for point in data_matrix])

    def _compute_edge_len(self, graph, data_matrix, target):
        _max_dist = self._diameter(data_matrix)
        for src_id, dest_id in graph.edges():
            if src_id != dest_id:
                px = data_matrix[src_id]
                pz = data_matrix[dest_id]
                dist = np.linalg.norm(px - pz)
                desired_dist = dist / _max_dist
                if desired_dist == 0:
                    desired_dist = _max_dist / float(data_matrix.shape[0])
                # if endpoints of an edge have the same
                # class then contract teh desired edge length
                if target[src_id] == target[dest_id]:
                    desired_dist /= (1 + self.class_bias)
                graph[src_id][dest_id]['len'] = desired_dist
                graph[src_id][dest_id]['weight'] = 1

                # if shift edge has one endpoint that is an
                # outlier then remove the edge altogether
                i_outl = graph.node[src_id]['outlier']
                j_outl = graph.node[dest_id]['outlier']
                if i_outl or j_outl:
                    if graph[src_id][dest_id]['edge_type'] == 'shift':
                        graph.remove_edge(src_id, dest_id)

        return graph

    def _kernel_shift_links(self, kernel_matrix=None,
                            density_matrix=None,
                            kernel_matrix_sorted_ids=None,
                            k_quick_shift=None,
                            target=None):
        num_targets = len(set(target))
        # if there are fewer targets than  k_quick_shift then reduce
        effective_threshold = min(k_quick_shift, num_targets)
        size = kernel_matrix.shape[0]
        # if a denser neighbor cannot be found then assign link to the
        # instance itself
        link_ids = list(range(size))
        # for all instances determine link link
        for i, densities in enumerate(density_matrix):
            i_density = densities[0]
            counter = 0
            classes = set()
            classes.add(target[i])
            # for all neighbors from the closest to the furthest
            for jj, j_density in enumerate(densities):
                j = kernel_matrix_sorted_ids[i, jj]
                if jj > 0:
                    # if the density of the neighbor is higher than the
                    # density of the instance and the class is different
                    # from previous classes then count +1
                    if j_density > i_density and target[j] not in classes:
                        counter += 1
                        classes.add(target[j])
                # proceed until counter reaches k_quick_shift
                if counter >= effective_threshold:
                    link_ids[i] = j
                    break
        return link_ids

    def _add_knn_links(self, graph, target, nneighbors_th=1,
                       kernel_matrix=None, kernel_matrix_sorted_ids=None):
        size = kernel_matrix.shape[0]
        for i in range(size):
            # add edges to the k nns with same class
            k = 0
            for jj in range(size):
                j = kernel_matrix_sorted_ids[i, jj]
                if i != j:
                    if target[i] == target[j]:
                        graph.add_edge(i, j, edge_type='knn', rank=k)
                        k += 1
                # after having added at most nneighbors_th links exit
                if k > nneighbors_th:
                    break
        return graph

    def _annotate_outliers(self, graph, nneighbors_th=1,
                           kernel_matrix=None, kernel_matrix_sorted_ids=None):
        size = kernel_matrix.shape[0]
        for i in range(size):
            counter = 0
            # add edges to the knns
            for jj in range(1, int(nneighbors_th) + 1):
                j = kernel_matrix_sorted_ids[i, jj]
                if i != j:
                    # check that within the k-nn also i is a knn of j
                    # i.e. use the symmetric nneighbor notion
                    upto = int(nneighbors_th) + 1
                    i_knns = kernel_matrix_sorted_ids[j, :upto]
                    if i in list(i_knns):
                        counter += 1
                        break
            if counter > 0:
                outlier_status = False
            else:
                outlier_status = True
            graph.node[i]['outlier'] = outlier_status

    def annotate_graph_with_graphviz_layout(self, graph, random_state=1):
        """Annotate graph with layout information."""
        # remove previous positional annotations
        for v in graph.nodes():
            if 'pos' in graph.node[v]:
                graph.node[v].pop('pos')

        layout_pos = nx.graphviz_layout(
            graph,
            prog='neato',
            args='-Goverlap=False -Gstart=%d' % random_state)

        # annotate nodes with positional information
        for v in graph.nodes():
            # check that no NaN is assigned as a coordinate
            if(layout_pos[v][0] == layout_pos[v][0] and
               layout_pos[v][1] == layout_pos[v][1]):
                pass
            else:
                print v, layout_pos[v], graph.neighbors(v)
                raise Exception('Assigned NaN value to position')
            graph.node[v]['pos'] = layout_pos[v]

    def display(self,
                target_dict=None,
                true_target=None,
                display=True,
                display_outliers=False,
                file_name='',
                cmap='rainbow',
                figure_size=15):
        """Display."""
        self.display_graph(
            self.instance_graph,
            target_dict=target_dict,
            true_target=true_target,
            display_hull=False,
            figure_size=figure_size,
            cmap=cmap,
            node_size=40,
            display_edge=False,
            display_outliers=display_outliers,
            file_name=file_name + '_1_clean.pdf')
        if display:
            plt.show()

    def display_links(self,
                      target_dict=None,
                      true_target=None,
                      display=True,
                      display_outliers=False,
                      file_name='',
                      cmap='rainbow',
                      figure_size=15):
        """display_links."""
        self.display_graph(
            self.instance_graph,
            target_dict=target_dict,
            true_target=true_target,
            display_edge=False,
            display_edges=True,
            display_hull=False,
            figure_size=figure_size,
            cmap=cmap,
            node_size=40,
            edge_thickness=.01,
            display_outliers=display_outliers,
            file_name=file_name + '_2_links.pdf')
        self.display_graph(
            self.instance_graph,
            target_dict=target_dict,
            true_target=true_target,
            display_edge=True,
            display_edges=False,
            display_hull=False,
            figure_size=figure_size,
            cmap=cmap,
            node_size=40,
            edge_thickness=.01,
            display_outliers=display_outliers,
            file_name=file_name + '_3_link.pdf')
        if display:
            plt.show()

    def display_hull(self,
                     target_dict=None,
                     true_target=None,
                     remove_outer_layer=False,
                     display=True,
                     display_outliers=False,
                     file_name='',
                     cmap='rainbow',
                     figure_size=15):
        """display_hull."""
        self.display_graph(
            self.instance_graph,
            target_dict=target_dict,
            true_target=true_target,
            display_hull=True,
            remove_outer_layer=remove_outer_layer,
            cmap=cmap,
            figure_size=figure_size,
            node_size=40,
            display_outliers=display_outliers,
            display_edge=True,
            file_name=file_name + '_4_hull.pdf')
        if display:
            plt.show()

    def display_graph(self,
                      graph,
                      target_dict=None,
                      true_target=None,
                      display_label=False,
                      display_edge=True,
                      display_edges=False,
                      display_outliers=False,
                      display_hull=True,
                      remove_outer_layer=False,
                      edge_thickness=40,
                      cmap='gist_ncar',
                      node_size=600,
                      figure_size=15,
                      file_name=''):
        """Display graph."""
        if target_dict is None:
            target_dict = {i: i for i in set(self.target)}
        fig, ax = plt.subplots(figsize=(figure_size, figure_size))

        if display_hull:
            patches = []
            for points in self._convex_hull(
                    self.embedded_data_matrix, self.target,
                    remove_outer_layer=remove_outer_layer):
                polygon = Polygon(points)
                patches.append(polygon)
            p = PatchCollection(patches,
                                cmap=plt.get_cmap(cmap),
                                alpha=0.2)
            p.set_array(np.array(range(len(set(self.target)))))
            ax.add_collection(p)
        layout_pos = self._get_node_layout_positions(graph)
        if true_target is not None:
            codes = true_target
        else:
            codes = np.array([graph.node[u]['group']
                              for u in graph.nodes()])
        instance_cols = self._get_node_colors(codes, cmap=cmap)
        if display_label:
            node_label_dict = {u: target_dict[u] for u in graph.nodes()}
            nx.draw_networkx_labels(graph, layout_pos, node_label_dict,
                                    font_size=14, font_weight='black',
                                    font_color='w')
            nx.draw_networkx_labels(graph, layout_pos, node_label_dict,
                                    font_size=14, font_weight='light',
                                    font_color='k')
        if display_outliers:
            outliers = [(u, col)
                        for u, col in zip(graph.nodes(), instance_cols)
                        if graph.node[u]['outlier']]
            nodelist = [u for u, col in outliers]
            node_color = [col for u, col in outliers]
            nx.draw_networkx_nodes(graph, layout_pos,
                                   node_color=node_color,
                                   nodelist=nodelist,
                                   node_size=node_size,
                                   alpha=.2,
                                   linewidths=0)
            nx.draw_networkx_nodes(graph, layout_pos,
                                   node_color=node_color,
                                   nodelist=nodelist,
                                   node_size=node_size / 2,
                                   alpha=.45,
                                   linewidths=0)
            non_outliers = [(u, col)
                            for u, col in zip(graph.nodes(), instance_cols)
                            if not graph.node[u]['outlier']]
            nodelist = [u for u, col in non_outliers]
            node_color = [col for u, col in non_outliers]
            nx.draw_networkx_nodes(graph, layout_pos,
                                   node_color=node_color,
                                   nodelist=nodelist,
                                   node_size=node_size,
                                   markeredgecolor='k',
                                   linewidths=1)
        else:
            nx.draw_networkx_nodes(graph, layout_pos,
                                   node_color=instance_cols,
                                   cmap=cmap, node_size=node_size,
                                   linewidths=1)
        if display_edges:
            # knn edges
            knn_edges = [(u, v) for u, v in graph.edges()
                         if graph[u][v].get('edge_type', '') == 'knn']
            knn_colors = [- graph[u][v]['rank'] for u, v in knn_edges]
            if knn_edges and knn_colors:
                nx.draw_networkx_edges(
                    graph, layout_pos, edgelist=knn_edges,
                    edge_cmap=plt.get_cmap('OrRd'),
                    edge_color=knn_colors, alpha=.4)
            # shift edges
            shift_edges = [
                (u, v) for u, v in graph.edges()
                if graph[u][v].get('edge_type', '') == 'shift']
            shift_colors = [-graph[u][v]['rank'] for u, v in shift_edges]
            if shift_edges and shift_colors:
                if len(set(shift_colors)) > 2:
                    nx.draw_networkx_edges(
                        graph, layout_pos, edgelist=shift_edges,
                        edge_cmap=plt.get_cmap('YlGnBu'),
                        edge_color=shift_colors, alpha=.2)
                else:
                    nx.draw_networkx_edges(
                        graph, layout_pos, edgelist=shift_edges,
                        edge_color='cornflowerblue', alpha=.2)
        if display_edge:
            # principal shift edges
            qs_th = 1
            shift_edges = [
                (u, v) for u, v in graph.edges()
                if graph[u][v].get('edge_type', '') == 'shift' and
                graph[u][v].get('rank', 0) == qs_th]
            if shift_edges:
                nx.draw_networkx_edges(
                    graph, layout_pos, edgelist=shift_edges, alpha=0.2,
                    width=1, edge_color='cornflowerblue')
        if display_hull:
            self._draw_class_id(graph,
                                target_dict=target_dict,
                                cmap=cmap,
                                node_size=600)
        plt.xticks([])
        plt.yticks([])
        plt.axis('off')
        plt.draw()
        if file_name:
            plt.savefig(file_name, bbox_inches='tight',
                        transparent=True, pad_inches=0)

    def _draw_class_id(self, graph,
                       target_dict=None,
                       cmap=None,
                       node_size=700):
        group_coords = defaultdict(list)
        node_label_dict = dict()
        ids = graph.nodes()
        for id in ids:
            group_id = graph.node[id]['group']
            group_coords[group_id].append(graph.node[id]['pos'])
        average_graph = nx.Graph()
        for group_id in group_coords:
            node_label_dict[group_id] = target_dict[group_id]
            coordinate_matrix = np.vstack(group_coords[group_id])
            coords = np.mean(coordinate_matrix, axis=0)
            average_graph.add_node(group_id, pos=coords)
        layout_pos = self._get_node_layout_positions(average_graph)
        codes = np.array([u for u in average_graph.nodes()])
        instance_cols = self._get_node_colors(codes, cmap=cmap)
        nx.draw_networkx_nodes(average_graph, layout_pos,
                               node_color=instance_cols,
                               cmap=cmap, node_size=node_size,
                               alpha=.5, linewidths=0)
        nx.draw_networkx_labels(average_graph, layout_pos, node_label_dict,
                                font_size=18, font_weight='black',
                                font_color='w')
        nx.draw_networkx_labels(average_graph, layout_pos, node_label_dict,
                                font_size=18, font_weight='light',
                                font_color='k')

    def _get_node_colors(self, codes, cmap=None):
        cm = matplotlib.cm.ScalarMappable(
            norm=matplotlib.colors.Normalize(vmin=min(codes),
                                             vmax=max(codes)),
            cmap=cmap)
        cols = np.array(range(min(codes), max(codes) + 1))
        # remove 4th column=alpha
        rgba_color_codes = cm.to_rgba(cols)[:, :3]

        enc = OneHotEncoder(sparse=False)
        codes = enc.fit_transform(codes.reshape(-1, 1))
        instance_cols = np.dot(codes, rgba_color_codes)[:, :3]
        return instance_cols

    def _get_node_layout_positions(self, graph):
        layout_pos = {v: graph.node[v]['pos'] for v in graph.nodes()}
        for pos in layout_pos:
            assert(layout_pos[pos][0] == layout_pos[pos][0] and
                   layout_pos[pos][1] == layout_pos[pos][1]),\
                'NaN position detected'
        return layout_pos

    def _convex_hull(self, data_matrix, target, remove_outer_layer=True):
        conv_hulls = []
        for c in set(target):
            points = []
            for z, xy in zip(target, data_matrix):
                if z == c:
                    points.append(xy)
            points = np.array(points)
            hull = ConvexHull(points)
            conv_hulls.append(points[hull.vertices])
        if remove_outer_layer:
            # remove conv hulls
            second_conv_hulls = []
            for c in set(target):
                points = []
                all_points = []
                for z, xy in zip(target, data_matrix):
                    if z == c:
                        all_points.append(xy)
                        if xy not in conv_hulls[c]:
                            points.append(xy)
                points = np.array(points)
                all_points = np.array(all_points)
                # if the points in the convex hull are more than half of the
                # total, then do not remove them from the set
                if len(points) <= 2:
                    hull = ConvexHull(all_points)
                    second_conv_hulls.append(all_points[hull.vertices])
                else:
                    hull = ConvexHull(points)
                    second_conv_hulls.append(points[hull.vertices])
            return second_conv_hulls
        else:
            return conv_hulls

max_stars_repo_name: rgerkin/EDeN

max_issues_repo_name: rgerkin/EDeN

max_forks_repo_name: rgerkin/EDeN

reason: import numpy,from scipy,import networkx


text:
import csv as csv
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import sys as sys
import traceback

from scipy import stats
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.model_selection import train_test_split
from sklearn.neighbors import KNeighborsClassifier
from sklearn.preprocessing import MinMaxScaler
from sklearn.preprocessing import Imputer
from sklearn.preprocessing import StandardScaler
import Helpers.DataFrameHelpers.DataFrameChecker as dfc
import Helpers.CheckingValueHelpers.CheckingValueHelper as cvh

class Preprocessing:
    """Preprocessing Class.

    """

    def __init__(self, df):
        self.df = df
        self.datetime_kinds = ['year', 'month', 'day', 'hour', 'minute', 'second', 'microsecond']

    def add_id(self):
        self.df = self.df.reset_index()
        self.df.info()
        return self.df

    def complement_with_median(self,column):
        """Complete 'NaN' data with median.

        This method completes 'NaN' data with median.

        Args:
            column (str): column name you want to fill.

        Returns:
            Dataframe: dataframe which is filled with median.
        """
        #median = self.df[column].dropna().median()
        #self.df[column] = self.df[column].fillna(median)
        #print("This method is not still tested")
        imr = Imputer(missing_values='NaN', strategy='median', axis=0)
        imr = imr.fit(self.df[[column]])
        self.df[column] = imr.transform(self.df[[column]]).ravel()
        return self.df

    def complement_with_mean(self,column):
        """Complete 'NaN' data with mean.

        This method completes 'NaN' data with mean.

        Args:
            column (str): column name you want to fill.

        Returns:
            Dataframe: dataframe which is filled with mean.
        """
        imr = Imputer(missing_values='NaN', strategy='mean', axis=0)
        imr = imr.fit(self.df[[column]])
        self.df[column] = imr.transform(self.df[[column]]).ravel()
        return self.df

    def complement_with_mostfrequency(self,column):
        """Complete 'NaN' data with most frequency value.

        Args:
            column (str): column name you want to fill.

        Returns:
            Dataframe: dataframe which is filled with most frequency value.
        """
        imr = Imputer(missing_values='NaN', strategy='most_frequent', axis=0)
        imr = imr.fit(self.df[[column]])
        self.df[column] = imr.transform(self.df[[column]]).ravel()
        return self.df

    def drop_colum(self, column):
        """Drop column

        Args:
            column(str): column you want to drop

        Returns:
            Dataframe: Dataframe which is dropped column.
        """
        try:
            self.df = self.df.drop(column, axis=1)
        except:
            print("Unexpected Exception")
            traceback.print_exc()
            sys.exit()
        return self.df

    def drop_nan(self):
        """Drop all 'NaN' rows.

        Returns:
            Dataframe: Dataframe which is removed all NaN rows.
        """
        self.df = self.df.dropna(axis=0, how='any')
        return self.df

    def drop_rows(self,column,term,threshold):
        """Drop rows

        Args:
            ans(str): term of rows which you want to drop

        Returns:
            Dataframe: Dataframe which is dropped rows.
        """
        try:
            if term == '=':
                self.df = self.df[self.df[column] == threshold]
            elif term == '<':
                self.df = self.df[self.df[column] < threshold]
            elif term == '>':
                self.df = self.df[self.df[column] > threshold]
        except TypeError:
            phelper.print_error("invalid type comparison")
        return self.df

    # xxxx-xx-xx xx:xx:xx to 0~7
    def convert_time_to_weekdays(self,column):
        """Convert time to weekdays.
        This method covnerts time to weekdays(0,1,2,3,4,5,6).

        Args:
            column(str): column you want to convert.
        Returns:
            Dataframe: dataframe with converted time to weekdays value(0: Monday - 6: Sunday).

        Examples:
            original data : ['2017-11-07 09:30:38']

        """
        try:
            self.df[column + '_weekday'] = pd.to_datetime(self.df[column]).dt.weekday
            self.df.info()
        except:
            print("Unexpected Exception")
            traceback.print_exc()
            sys.exit()
        return self.df

    # xxxx-xx-xx xx:xx:xx to 0~24
    def convert_time_to_datetime(self,column,time_kind):
        """Convert time to date time.

        Args:
            column (str): column you want to convert.

        Returns:
            Dataframe: dataframe with converted time to time.

        Examples:
            original data : ['2017-11-07 09:30:38']

            year: (ex),[2017]
            month:(ex),[11]
            day: (ex),[7]
            hour: (ex),[9]
            minute: (ex),[30]
            second; (ex),[38]
            microsecond (ex),[0]
        """
        if not time_kind in self.datetime_kinds :
            raise(NotSupportedError)

        new_column_name = column + '_' + time_kind
        exec('self.df[new_column_name] = pd.to_datetime(self.df[column_name]).dt.' + time_kind)
        self.df.info()
        return self.df

    def fill_in_NaN(self,column,value):
        """Fill in 'NaN' values in column with value.

        Args:
            column (str): column name.
            value :value you want to fill in NaN with.

        Returns:
            Dataframe: dataframe.

        """
        self.df[column] = self.df[column].fillna(value)
        return self.df

    # It is uede for embarked to Int {S:0 C:1 Q:2}
    def convert_object_to_int(self,column):
        """Convert string(object) to integer in column.

        Args:
            column (str): column name.

        Returns:
            Dataframe: dataframe with converted column.

        Examples:
            S:0
            C:1
            Q:2
        """
        print(self.df[column].value_counts())

        coi_dict = {}
        u = self.df[column].unique()
        for i,j in enumerate(u):
            print("convert " + j + ":" + str(i))
            coi_dict[j] = i
        self.df["converted_" + column] = self.df[column].map(coi_dict).astype(int)

        return self.df

    #正規化するメソッド
    def convert_data_to_nomalized(self,column_name):
        if type(column_name) is not str:
            return False
        mms = MinMaxScaler()
        self.df[column_name] = mms.fit_transform(self.df[column_name])
        return self.df

    #Add two columns
    def sum_columns(self,column_name,column_name2,new_column_name):
        self.df[new_column_name] = self.df[column_name] + self.df[column_name2]
        return self.df

    def sample(self,n):
        try:
            return self.df.sample(n)
        except TypeError:
            phelper.PrintHelper.print_error("Type Error:")
            return self.df

    #convert value to boolean
    def convert_columns_to_boolean(self,column_name,new_column_name,sleshhold):
        self.df[new_column_name] = self.df[column_name].apply(lambda x: 1 if x < sleshhold else 0)
        return self.df

    def delete_outlier(self,column_name,bias):
        q1 = self.df[column_name].quantile(.25)
        q3 = self.df[column_name].quantile(.75)
        iqr = q3 - q1

        outlier_min = q1 - (iqr) * bias
        outlier_max = q3 + (iqr) * bias

        deleted_df = self.df[(outlier_min < self.df[column_name]) & (self.df[column_name] < outlier_max)]
        return deleted_df

    def convert_standard_scaler(self):
        """Convert dataframe to standard scale data

        Returns:
            Dataframe: dataframe with converted column.

        Examples:

        Returns:
            Dataframe: standaard scaled dataframe
        """

        sc = StandardScaler()
        return sc.fit_transform(self.df)

class NotSupportedError(Exception):
    pass

ext: py

max_stars_repo_name: hirohio/Hello-World-ML

max_issues_repo_name: hirohio/Hello-World-ML

max_forks_repo_name: hirohio/Hello-World-ML

reason: import numpy,from scipy


text:
#!/usr/bin/env python
# -*- coding: utf-8
# pytest unit tests for spinalcordtoolbox.deepseg

import os

import pytest
import numpy as np
import nibabel

import spinalcordtoolbox as sct
from spinalcordtoolbox.utils import sct_test_path
import spinalcordtoolbox.deepseg.models

from spinalcordtoolbox.scripts import sct_deepseg

def test_install_model():
    """
    Download all models, to allow subsequent tests on the model packages.
    :return:
    """
    for name_model, value in sct.deepseg.models.MODELS.items():
        if value['default']:
            sct.deepseg.models.install_model(name_model)
            # Make sure all files are present after unpacking the model
            assert sct.deepseg.models.is_valid(sct.deepseg.models.folder(name_model))

def test_model_dict():
    """
    Make sure all fields are present in each model.
    :return:
    """
    for key, value in sct.deepseg.models.MODELS.items():
        assert('url' in value)
        assert('description' in value)
        assert('default' in value)

# noinspection 801,PyShadowingNames
@pytest.mark.parametrize('fname_image, fname_seg_manual, fname_out, task', [
    (sct_test_path('t2s', 't2s.nii.gz'),
     sct_test_path('t2s', 't2s_seg-deepseg.nii.gz'),
     't2s_seg_deepseg.nii.gz',
     'seg_sc_t2star'),
])
def test_segment_nifti(fname_image, fname_seg_manual, fname_out, task,
                       tmp_path):
    """
    Uses the locally-installed sct_testing_data
    """
    fname_out = str(tmp_path/fname_out)  # tmp_path for automatic cleanup
    sct_deepseg.main(['-i', fname_image, '-task', task, '-o', fname_out, '-thr', str(0.9)])
    # TODO: implement integrity test (for now, just checking if output segmentation file exists)
    # Make sure output file exists
    assert os.path.isfile(fname_out)
    # Compare with ground-truth segmentation
    assert np.all(nibabel.load(fname_out).get_fdata() ==
                  nibabel.load(fname_seg_manual).get_fdata()[..., 0])

max_stars_repo_licenses: Unlicense MIT

max_issues_repo_licenses: Unlicense MIT

max_forks_repo_licenses: Unlicense MIT

reason: import numpy


text:
import os.path
import shutil
import statistics
from collections import defaultdict
from datetime import datetime, timedelta

import matplotlib.pyplot as plt
import networkx as nx
import numpy as np
import torch
import torch_geometric
from torch.nn.utils.rnn import pack_padded_sequence
from torch_geometric.data import Data
from torch_geometric.utils import remove_self_loops, to_undirected

import cargonet.preprocessing.datalake.retrieval as retrieval
import cargonet.preprocessing.graphs.tgraph as tgraph
from cargonet.dataset.dataset import RailDataset
from cargonet.models.utils import encode_datetime_cyclical
from cargonet.utils.geo import GERMANY_BBOX_1
from cargonet.utils.pdf import concat_pdfs

class Encoder:

    def __init__(self, first_time, seq_len, pred_seq_len, max_transports, net):
        self.first_time = first_time
        self.seq_len = seq_len
        self.pred_seq_len = pred_seq_len

        self.net, self.net_mapping = net
        self.static_net_size = len(self.net_mapping)
        assert (
            self.static_net_size - 1 == np.array(list(self.net_mapping.values())).max()
        )

        self.conflicting_temp = defaultdict(list)
        self.route_node_mapping = dict()
        self.transports_node_lut = defaultdict(list)
        self.transports = []
        self.transport_mask = torch.ones(max_transports, dtype=torch.long) * -1
        self.acc = nx.DiGraph()
        self.max_transports = max_transports

        n_edges_temp = (seq_len + pred_seq_len - 1) * max_transports  # unidirectional
        n_edges_spat = (seq_len + pred_seq_len) * max_transports * 2  # bidirectional

        self.ground_edge_index = torch.zeros(
            max_transports, seq_len + pred_seq_len, 2, dtype=torch.long
        )
        self.temporal_edge_index = torch.zeros(
            max_transports, seq_len + pred_seq_len - 1, 2, dtype=torch.long
        )

        self.temporal_edge_attr = torch.zeros(
            max_transports,
            seq_len + pred_seq_len - 1,
            len(self.route_edge_fts),
            dtype=torch.long,
        )

        self.y_pred = torch.zeros(max_transports, pred_seq_len, len(self.predict_fts))
        self.x = torch.zeros(
            max_transports, seq_len + pred_seq_len, len(self.seq_route_node_fts)
        )
        self.current_transports = torch.zeros(
            max_transports, seq_len + pred_seq_len, dtype=torch.bool
        )
        self.routes = torch.zeros(
            max_transports, seq_len + pred_seq_len, dtype=torch.long
        )

class RoadsAsEdgesV1(Encoder):

    common_node_fts = [
        "pause",
        "arrival_rel",
        "departure_rel",
        "month_sin",
        "month_cos",
        "arrival_sin",
        "arrival_cos",
    ]
    seq_route_node_fts = common_node_fts + ["delay"]
    horizon_route_node_fts = common_node_fts
    station_node_fts = ["lat", "lon"]
    predict_fts = ["delay"]

    station_edge_fts = []
    route_edge_fts = ["distance", "plannedDuration", "duration"]

    def add(self, t, transport, center_edge, desc, anc):
        cu, cv = center_edge
        desc_g = transport.subgraph(desc + [cv])
        anc_g = transport.subgraph(anc + [cu])
        route_g = transport.subgraph(anc + [cu, cv] + desc)

        transport_ids = list(
            set([data.get("transportId") for _, data in route_g.nodes(data=True)])
        )
        assert len(transport_ids) == 1
        transport_id = transport_ids[0]

        if not transport_id in self.route_node_mapping:
            self.route_node_mapping[transport_id] = len(self.route_node_mapping)
        t = self.route_node_mapping[transport_id]
        self.transport_mask[t] = transport_id

        pad_seq = self.seq_len - len(anc_g)

        # Common node features
        for seq, n in enumerate(nx.topological_sort(route_g)):
            stationId = transport.nodes[n].get("stationId")
            plannedArrTime = transport.nodes[n].get("plannedArrivalTime")
            plannedDepTime = transport.nodes[n].get("plannedDepartureTime")

            self.x[t, pad_seq + seq, 0] = (
                plannedDepTime - plannedArrTime
            ).total_seconds() / 60.0  # Pause in minutes

            self.x[t, pad_seq + seq, 1] = (
                plannedArrTime - self.first_time
            ).total_seconds() / 60.0  # Planned Arrival in absolute delta minutes
            self.x[t, pad_seq + seq, 2] = (
                plannedDepTime - self.first_time
            ).total_seconds() / 60.0  # Planned Departure in absolute delta minutes

            # Arrival in cyclical
            cyc_p_arr_hr, cyc_p_arr_month = encode_datetime_cyclical(plannedArrTime)

            # Month
            cyc_p_arr_month_sin, cyc_p_arr_month_cos = cyc_p_arr_month
            self.x[t, pad_seq + seq, 3] = cyc_p_arr_month_sin
            self.x[t, pad_seq + seq, 4] = cyc_p_arr_month_cos

            cyc_p_arr_hr_sin, cyc_p_arr_hr_cos = cyc_p_arr_hr
            self.x[t, pad_seq + seq, 5] = cyc_p_arr_hr_sin
            self.x[t, pad_seq + seq, 6] = cyc_p_arr_hr_cos

            # Departure in cyclical (assume month is the same)
            cyc_p_arr_hr, _ = encode_datetime_cyclical(plannedArrTime)

            self.current_transports[t, pad_seq + seq] = 1
            self.routes[t, pad_seq + seq] = stationId

            this_edge_index = (self.seq_len + self.pred_seq_len) * t + (pad_seq + seq)
            station_node = self.net_mapping.get(stationId, this_edge_index)
            self.ground_edge_index[t, pad_seq + seq] = torch.LongTensor(
                [this_edge_index, station_node]
            )

            station_node = self.net_mapping.get(stationId)
            self.conflicting_temp[station_node].append(this_edge_index)

        # Sequence node features
        for seq, n in enumerate(nx.topological_sort(anc_g)):
            self.x[t, pad_seq + seq, len(self.common_node_fts)] = transport.nodes[
                n
            ].get("delay")

        # Horizon node features
        for seq, n in enumerate(nx.topological_sort(desc_g)):
            # Predictions
            self.y_pred[t, seq, 0] = transport.nodes[n].get("delay")

        for seq, edge in enumerate(route_g.edges):
            try:
                self.temporal_edge_index[t, pad_seq + seq] = torch.LongTensor(
                    [
                        (self.seq_len + self.pred_seq_len) * t + (pad_seq + seq),
                        (self.seq_len + self.pred_seq_len) * t + (pad_seq + seq + 1),
                    ]
                )

                self.temporal_edge_attr[t, pad_seq + seq, 0] = (
                    transport.edges[edge].get("distance") / 1000.0
                )
                self.temporal_edge_attr[t, pad_seq + seq, 1] = (
                    transport.edges[edge].get("plannedDuration").total_seconds() / 60.0
                )
                self.temporal_edge_attr[t, pad_seq + seq, 2] = (
                    transport.edges[edge].get("duration").total_seconds() / 60.0
                )
            except Exception as e:
                print("Error while adding temp edge attrs", str(e))

    def encode(self, t):
        self.spatial_edge_index = []
        for station, conflicting in self.conflicting_temp.items():
            # Add edge between pair of conflicting nodes in each direction
            for c1 in conflicting:
                for c2 in conflicting:
                    if c1 == c2:
                        continue
                    se = torch.LongTensor([c1, c2])
                    self.spatial_edge_index.append(se)

        if len(self.spatial_edge_index) > 0:
            self.spatial_edge_index = (
                torch.stack(self.spatial_edge_index).t().contiguous()
            )
        else:
            self.spatial_edge_index = torch.zeros(2, 0, dtype=torch.long)

        self.temporal_edge_index = self.temporal_edge_index.view(-1, 2).t().contiguous()

        self.ground_edge_index = self.ground_edge_index.view(-1, 2).t().contiguous()
        if self.ground_edge_index.size(1) > 0:
            self.ground_edge_index[0, :] += self.net.x.size(0)

        num_transports = (self.current_transports == 1).sum(dim=1)
        num_transports = num_transports.numpy()
        if num_transports.size < 1:
            return Data(), self.acc

        num_transports = num_transports.max()

        temporal_node_mask = torch.BoolTensor([1] * self.x.size(0))
        spatial_node_mask = ~temporal_node_mask

        data = Data(
            x=self.x,
            y=self.y_pred,
            t=t.timestamp(),
            first_time=self.first_time.timestamp(),
            ground_edge_index=self.ground_edge_index,
            spatial_edge_index=self.spatial_edge_index,
            temporal_edge_index=self.temporal_edge_index,
            temporal_edge_attr=self.temporal_edge_attr,
            temporal_node_mask=temporal_node_mask,
            spatial_node_mask=spatial_node_mask,
            transport_mask=self.transport_mask,
            current_transports=self.current_transports,
            routes=self.routes,
            num_transports=torch.IntTensor([num_transports]),
        )

        return data, self.acc

class ActiveRoutesV1(RailDataset):
    """
    Dataset for predicting average delay on transport edges
    """

    node_feature_mapping = ["stationId", "imId", "country", "lat", "lon"]
    edge_feature_mapping = ["distance", "popularity"]

    def __init__(
        self,
        root,
        name=None,
        transform=None,
        pre_transform=None,
        limit=1,
        plot_download=False,
        plot_processing=False,
        force_reprocess=False,
        force_redownload=False,
        normalize_net=True,
        verbose=True,
        lookbehind_steps=3,
        lookahead_steps=1,
        lookahead=timedelta(hours=1),
        lookbehind=timedelta(minutes=10),
        interval=timedelta(minutes=10),
        batch=timedelta(hours=24),
        padding=timedelta(hours=0),
        bbox=None,
        seq_interval=1,
        pred_interval=1,
        seq_len=3,
        pred_seq_len=3,
    ):
        self.seq_len = seq_len
        self.pred_seq_len = pred_seq_len
        super().__init__(
            root=root,
            name=name,
            transform=transform,
            pre_transform=pre_transform,
            limit=limit,
            plot_download=plot_download,
            plot_processing=plot_processing,
            force_reprocess=force_reprocess,
            force_redownload=force_redownload,
            normalize_net=normalize_net,
            verbose=verbose,
            lookbehind_steps=lookbehind_steps,
            lookahead_steps=lookahead_steps,
            lookahead=lookahead,
            lookbehind=lookbehind,
            interval=interval,
            batch=batch,
            padding=padding,
            bbox=bbox,
            full_route=True,
            unique_index=True,
            undirected=False,
        )

    def download(self):
        super().download()

    @property
    def processed_file_names(self):
        return ["processed_%d.pt" % f for f in range(0, len(self.raw_file_names))]

    def debug_plot(
        self, t, state, prefix=None, bbox=GERMANY_BBOX_1, labels=True, size=3,
    ):
        filename = "%s/%s%s%s.pdf" % (
            self.name,
            f"{prefix}/" if prefix else "",
            prefix or "",
            t if not isinstance(t, datetime) else self.format_timestamp(t),
        )
        # TODO: need new kind of plot
        if False:
            NXTransportPlot(
                state,
                filename=filename,
                show=False,
                node_labels=False,
                edge_labels=labels,
                thickness=size,
                node_size=size,
                colorbar_range=(-1.0, 3.0),  # (-100, 300),
                bbox=bbox,
                cities=True,
                check=False,
                delay=True,
                cmap=opaque_delay_cmap if opaque else delay_cmap,
                subtitle=f"{prefix} at {str(t)}" if prefix else str(t),
            ).plot()

    @property
    def encoder(self):
        return RoadsAsEdgesV1

    def aggregate(self, t, state):
        acc = nx.DiGraph()
        ustate = state.to_undirected()

        def debug(g, ref=None):
            pos = nx.spring_layout(ref if ref else g)
            if ref:
                nx.draw(ref, pos, node_size=200, node_color="green", arrows=True)
            labels = nx.get_node_attributes(g, "arrivalTime")
            nx.draw(g, pos, labels=labels, node_size=150, node_color="red", arrows=True)
            plt.show()

        # extract all subgraphs
        active_transports = [state.subgraph(c) for c in nx.connected_components(ustate)]
        print("%d transports" % len(active_transports))

        def get_segment(_transport, _t):
            # Assumptions
            if not all(
                [
                    not (_transport.has_edge(u, v) and _transport.has_edge(v, u))
                    for u, v in _transport.edges
                ]
            ):
                # This can happen if a transport uses a segment twice
                pass

            # Find segment of current timestep t
            current_segment = None
            for u, v, data in _transport.edges(data=True):
                src = _transport.nodes[u].get("arrivalTime")
                dest = _transport.nodes[v].get("arrivalTime")
                if src <= _t < dest:
                    current_segment = (u, v)
            return current_segment

        # Filter only transports that fit current timeframe
        first_time = None
        active_transports = [
            tp for tp in active_transports if get_segment(tp, t) is not None
        ]
        for transport in active_transports:
            for _, data in transport.nodes(data=True):
                candidates = [
                    data.get("plannedArrivalTime"),
                    data.get("arrivalTime"),
                    data.get("plannedDepartureTime"),
                    data.get("departureTime"),
                ]
                candidate = min(candidates)
                if first_time is None:
                    first_time = candidate
                first_time = min(candidate, first_time)

        encoder = self.encoder(
            first_time=first_time,
            seq_len=self.seq_len,
            pred_seq_len=self.pred_seq_len,
            max_transports=len(active_transports),
            net=(self.net, self.net_mapping),
        )

        for transport in active_transports:
            current_segment = get_segment(transport, t)
            if current_segment is None:
                # Does not fit current timeframe
                continue
            cu, cv = current_segment
            # Include some ancestors and descendants
            desc = list(nx.nodes(nx.dfs_tree(transport, cv)))[: self.pred_seq_len]
            anc = list(nx.nodes(nx.dfs_tree(transport.reverse(copy=True), cu)))[
                : self.seq_len
            ]
            assert len(set(desc + anc)) <= self.pred_seq_len + self.seq_len + 1

            debug_sg = desc + anc + [cu, cv]
            if False:
                debug(state.subgraph(debug_sg), ref=transport)

            encoder.add(t, transport, center_edge=(cu, cv), anc=anc, desc=desc)

        return encoder.encode(t)

    def process(self, skip=False):
        states_count = len(self.raw_paths)
        sl, pl = (
            self.seq_len,
            self.pred_seq_len,
        )
        for i in range(states_count):
            out = os.path.join(
                self.processed_dir, self.processed_file_names[i], # - sl - pl],
            )

            # Check if already processed
            if skip:
                if i < 28_400:
                    print("Skipping ", i)
                    continue
                if os.path.isfile(out):
                    print("Skipping ", i)
                    continue

            # Read transport state at time step t and some previous steps
            state = nx.read_gpickle(self.raw_paths[i])
            t = self.timerange[i]
            t_next = t + self.interval
            self.vlog(
                "Processing t[%d] %s - %s (%d/%d, %d+%d states)" % (i, t, t_next, i, states_count, sl, pl,)
            )

            data, graph = self.aggregate(t, state)
            # print(data)

            # Plot combined graph
            if self.plot_processing:
                self.debug_plot(i, graph, prefix="combined", size=1, labels=False)

            # Apply filters and transformations
            if self.pre_filter is not None and not self.pre_filter(data):
                continue

            if self.pre_transform is not None:
                data = self.pre_transform(data)

            torch.save(
                data,
                out,
            )

def build_dataset(limit, plot, rebuild=False, reprocess=False):
    dataset_name = "active-routes-dataset-v1"

    base_path = os.path.dirname(os.path.realpath(__file__))
    base_dataset_path = os.path.join(base_path, "../../datasets")
    assert os.path.exists(base_dataset_path)
    dataset_path = os.path.join(base_dataset_path, dataset_name)

    try:
        dataset = ActiveRoutesV1(
            root=dataset_path,
            name=dataset_name,
            plot_processing=plot,
            limit=limit,
            batch=timedelta(hours=2),
            force_redownload=rebuild,
            force_reprocess=reprocess,
        )
        print(dataset)
        print(len(dataset))
    except Exception as e:
        raise
        print(e)

    if plot:
        print("Combining plots")
        base_fig_path = os.path.join(base_path, "../../fig")
        concat_pdfs(
            source_dir=os.path.join(base_fig_path, dataset_name),
            out_file=os.path.join(base_fig_path, "%s.pdf" % dataset_name),
        )

max_stars_repo_name: romnnn/rail-stgcnn

max_issues_repo_name: romnnn/rail-stgcnn

max_forks_repo_name: romnnn/rail-stgcnn

reason: import numpy,import networkx


text:
import os
import random
import pytest
import pathlib
import tempfile
import numpy as np
import pandas as pd
import pandas.api.types as ptypes

from cytominer_eval.transform import metric_melt
from cytominer_eval.operations import replicate_reproducibility

random.seed(123)
tmpdir = tempfile.gettempdir()

example_file = "SQ00015054_normalized_feature_select.csv.gz"
example_file = pathlib.Path(
    "{file}/../../example_data/compound/{eg}".format(
        file=os.path.dirname(__file__), eg=example_file
    )
)

df = pd.read_csv(example_file)

meta_features = [x for x in df.columns if x.startswith("Metadata_")]
features = df.drop(meta_features, axis="columns").columns.tolist()

similarity_melted_df = metric_melt(
    df=df,
    features=features,
    metadata_features=meta_features,
    similarity_metric="pearson",
)

def test_replicate_reproducibility():
    replicate_groups = ["Metadata_broad_sample", "Metadata_mg_per_ml"]
    output = replicate_reproducibility(
        similarity_melted_df=similarity_melted_df,
        replicate_groups=replicate_groups,
        quantile_over_null=0.95,
    )
    expected_result = 0.4583

    assert np.round(output, 4) == expected_result

    replicate_groups = ["Metadata_moa"]
    output = replicate_reproducibility(
        similarity_melted_df=similarity_melted_df,
        replicate_groups=replicate_groups,
        quantile_over_null=0.95,
    )
    expected_result = 0.3074

    assert np.round(output, 4) == expected_result

def test_replicate_reproducibility_uniquerows():
    with pytest.raises(AssertionError) as err:
        replicate_groups = ["Metadata_pert_well"]
        output = replicate_reproducibility(
            similarity_melted_df=similarity_melted_df,
            replicate_groups=replicate_groups,
            quantile_over_null=0.95,
        )
    assert "no replicate groups identified in {rep} columns!".format(
        rep=replicate_groups
    ) in str(err.value)

def test_replicate_reproducibility_return_cor():
    # Confirm that it works with multiple columns
    replicate_groups = ["Metadata_moa"]

    output, med_cor = replicate_reproducibility(
        similarity_melted_df=similarity_melted_df,
        replicate_groups=replicate_groups,
        quantile_over_null=0.95,
        return_median_correlations=True,
    )
    expected_result = 0.3074

    assert np.round(output, 4) == expected_result
    assert (
        med_cor.sort_values(
            by="similarity_metric", ascending=False
        ).Metadata_moa.values[0]
        == "PKC activator"
    )
    assert np.round(med_cor.similarity_metric.max(), 4) == 0.9357

    replicate_groups = ["Metadata_broad_sample", "Metadata_mg_per_ml"]
    output, med_cor = replicate_reproducibility(
        similarity_melted_df=similarity_melted_df,
        replicate_groups=replicate_groups,
        quantile_over_null=0.95,
        return_median_correlations=True,
    )
    expected_result = 0.4583

    assert np.round(output, 4) == expected_result
    assert np.round(med_cor.similarity_metric.mean(), 4) == 0.5407

max_stars_repo_name: michaelbornholdt/cytominer-eval

max_issues_repo_name: michaelbornholdt/cytominer-eval

max_forks_repo_name: michaelbornholdt/cytominer-eval


text:
from collections import Counter
from numpy.linalg import norm
from math import log
from keyword import kwlist
import os
import re

def makeCorpus(root_folder):
  """
  Args:
    root_folder (String): root path of the corpus

  Returns:
    corpus (dict): a dictionary contains term frequency of every experiments
      the key is the experiment's name, the value is a tuple of two lists.
      1st list named 'words' contains all words in the experiments.
      2nd list named 'vector' contains the it-idf vector.
  """
  pattern = re.compile('[\\w\\d]+')
  corpus = {}
  for exp in os.listdir(root_folder):
    path = root_folder + exp
    if not os.path.isdir(path):
      continue
    content = catPythonScript(path)
    # filter out python keywords and find each words using regex
    words = filter(lambda x: x not in kwlist, re.findall(pattern, content))

    # histogram = { word : how many times the word appears }
    histogram = Counter(words)
    words = histogram.keys()
    values = histogram.values()

    # divided by document length to get term frequency
    sum_ = sum(values)
    vector = map(lambda x: float(x)/sum_, values)
    corpus[exp] = (words, vector)

  # for now, the corpus only has term frequency vectors.
  # calculateTFIDF will turn tf vectors into tf-idf vectors.
  calculateTFIDF(corpus)
  return corpus

def catPythonScript(folder):
  """
  Args:
    folder (String): the target folder

  Returns:
    cat (String): content of all *.py file under the target folder
  """
  cat = ''
  for fn in [folder + '/' + d for d in os.listdir(folder)]:
    content = None
    if os.path.isdir(fn):
      content = catPythonScript(fn)
    if fn.endswith('.py'):
      content = open(fn).read()
    if content:
      cat += content
  return cat

def calculateWordIDF(word, corpus):
  """
  Args:
    word (String): the target word to calculate IDF value
    corpus (dict): in order to count how many documents the word appears

  Note:
    in Information Retrieval, a corpus contains several documents.
    in here, a experiment is being treated as a document in IR.

  Returns:
    (float): IDF value of the word under given corpus
  """
  # IDF = log( how many documents the corpus has / (1 + how many documents contain the word) )
  df = 1.0
  for exp in corpus.keys():
    words, _ = corpus[exp]
    if word in words: df += 1
  return log(len(corpus) / df)

def calculateTFIDF(corpus):
  """
  Args:
    corpus (dict): a term frequency corpus

  Returns:
    (None): vectors are modified directly in the corpus dictionary
  """
  for exp in corpus.keys():
    words, tfs = corpus[exp]
    for idx, word in enumerate(words):
      idf = calculateWordIDF(word, corpus)
      tfs[idx] *= idf

max_stars_repo_name: fatcloud/PyCV-Climbing-Wall

max_issues_repo_name: fatcloud/PyCV-Climbing-Wall

max_forks_repo_name: fatcloud/PyCV-Climbing-Wall

reason: from numpy


text:
import numpy as np

class Apriori():
    def __init__(self, data):
        self.data = data
        self.treshold = 1 / 2 * len(data[next(iter(data))])

    def apriory(self, data):
        return self.isRepresentSameThing(self.selectSignificent(self.itemSet(data)))

    def frequency(self, data):
        frequency = {}
        for key in data:
            frequency[key] = (sum(data[key]))

        return frequency

    def selectSignificent(self, data):
        # frequency = self.frequency(data)
        temp = {}
        for key in data:

            if data[key] >= self.treshold:
                temp[key] = data[key]

        data = temp
        return data

    def itemSet(self, data):
        temp = {}
        for k in data:
            for l in data:
                if k != l:
                    # print(k, data[k], l, data[l])
                    # temp[k + '&' + l] = temp.get(k + '&' + l, 0) + 1
                    isContinue = True
                    if k + '&' + l.split('&')[-1] in data:
                        isContinue = False
                    if l.split('&')[-1] + '&' + k in data:
                        isContinue = False
                    for s in range(len(k.split('&')) - 1):
                        if k.split('&')[s] != l.split('&')[s]:
                            isContinue = False
                    if isContinue:
                        temp[k + '&' + l.split('&')[-1]] = int(sum(self.search(*k.split('&'), *l.split('&'))))
        return self.selectSignificent(temp)
        # return temp

    def isRepresentSameThing(self, data):
        temp = {}
        for key in data:
            isContinue = True
            tempSum = sum(bytearray(key.encode('utf8')))
            for l in temp:
                if sum(bytearray(l.encode('utf8'))) == tempSum:
                    isContinue = False
            if isContinue:
                temp[key] = data[key]

        return temp

    def listXor(self, list1, list2):
        temp = np.zeros(len(list1))
        for i in range(len(list1)):
            if (list1[i] == list2[i]) and list1[i] == 1:
                temp[i] = 1

        return temp

    def search(self, *args):
        temp = np.ones(len(self.data[next(iter(self.data))]))
        for key in args:
            temp = self.listXor(self.data[key], temp)
        return temp

if __name__ == '__main__':
    testData = {
        'apple': [1, 1, 0, 1],
        'pinapple': [0, 1, 1, 1],
        'pair': [0, 0, 0, 1],
        'melon': [1, 0, 0, 1],
        'cherry': [1, 0, 1, 1]
    }
    test = Apriori(testData)
    print(test.apriory(testData))
    print(test.apriory(test.apriory(testData)))

max_stars_repo_name: furkankykc/Apriory-Algorithm

max_issues_repo_name: furkankykc/Apriory-Algorithm

max_forks_repo_name: furkankykc/Apriory-Algorithm


text:
# -*- coding: utf-8 -*-
"""
@author: hongyuan
"""

import numpy as np
import random
import torch
import torch.nn as nn
import torch.nn.functional as F

from cont_time_cell import CTLSTMCell

from torch.distributions.multinomial import Multinomial
class NeuralHawkes(nn.Module):

    def __init__(self, *,
        event_num, group_num,
        hidden_dim=32, beta=1.0, device=None):
        super(NeuralHawkes, self).__init__()

        self.event_num = event_num
        self.group_num = group_num
        self.hidden_dim = hidden_dim

        self.idx_BOS = {}
        for i in range(self.group_num):
            self.idx_BOS[i] = self.event_num + i
        self.idx_EOS = self.event_num + self.group_num
        self.idx_PAD = self.event_num + self.group_num + 1

        self.beta = beta

        device = device or 'cpu'
        self.device = torch.device(device)

        self.Emb = nn.Embedding(
            self.event_num + self.group_num + 2, self.hidden_dim)

        self.rnn_cell = CTLSTMCell(
            self.hidden_dim, beta=self.beta, device=self.device)

        self.hidden_lambda = nn.Linear(
            self.hidden_dim, self.event_num, bias=False)

        self.init_h = torch.zeros(size=[hidden_dim],
                                  dtype=torch.float32, device=self.device)
        self.init_c = torch.zeros(size=[hidden_dim],
                                  dtype=torch.float32, device=self.device)
        self.init_cb = torch.zeros(size=[hidden_dim],
                                   dtype=torch.float32, device=self.device)

        self.eps = np.finfo(float).eps
        self.max = np.finfo(float).max

    def cuda(self, device=None):
        device = device or 'cuda:0'
        self.device = torch.device(device)
        assert self.device.type == 'cuda'
        super().cuda(self.device)

    def cpu(self):
        self.device = torch.device('cpu')
        super().cuda(self.device)

    def getStates(self, event, dtime):
        batch_size, num_groups, T_plus_2 = event.size()
        cell_t_i_minus = self.init_c.unsqueeze(0).unsqueeze(0).expand(
            batch_size, num_groups, self.hidden_dim)
        cell_bar_im1 = self.init_cb.unsqueeze(0).unsqueeze(0).expand(
            batch_size, num_groups, self.hidden_dim)
        hidden_t_i_minus = self.init_h.unsqueeze(0).unsqueeze(0).expand(
            batch_size, num_groups, self.hidden_dim)

        all_cell, all_cell_bar = [], []
        all_gate_output, all_gate_decay = [], []
        all_hidden = []
        all_hidden_after_update = []
#把event一个一个放入
        for i in range(T_plus_2 - 1):
            # only BOS to last event update LSTM
            # <s> CT-LSTM
#256,1,16
            emb_i = self.Emb(event[:, :, i ])
            dtime_i = dtime[:, :, i + 1 ] # need to carefully check here

            cell_i, cell_bar_i, gate_decay_i, gate_output_i = self.rnn_cell(
                emb_i, hidden_t_i_minus, cell_t_i_minus, cell_bar_im1
            )
            #执行公式7
            _, hidden_t_i_plus = self.rnn_cell.decay(
                cell_i, cell_bar_i, gate_decay_i, gate_output_i,
                torch.zeros_like(dtime_i)
            )
            cell_t_ip1_minus, hidden_t_ip1_minus = self.rnn_cell.decay(
                cell_i, cell_bar_i, gate_decay_i, gate_output_i,
                dtime_i
            )
            all_cell.append(cell_i)
            all_cell_bar.append(cell_bar_i)
            all_gate_decay.append(gate_decay_i)
            all_gate_output.append(gate_output_i)
            all_hidden.append(hidden_t_ip1_minus)
            all_hidden_after_update.append(hidden_t_i_plus)
            cell_t_i_minus = cell_t_ip1_minus
            cell_bar_im1 = cell_bar_i
            hidden_t_i_minus = hidden_t_ip1_minus
            # </s> CT-LSTM
        # these tensors shape : batch_size, num_groups, T+1, hidden_dim
        # cells and gates right after BOS, 1st event, ..., N-th event
        # hidden right before 1st event, ..., N-th event, End event (PAD)
        all_cell = torch.stack( all_cell, dim=2)
        all_cell_bar = torch.stack( all_cell_bar, dim=2)
        all_gate_decay = torch.stack( all_gate_decay, dim=2)
        all_gate_output = torch.stack( all_gate_output, dim=2)
        all_hidden = torch.stack( all_hidden, dim=2 )
        all_hidden_after_update = torch.stack( all_hidden_after_update, dim=2)
        #assert all_gate_decay.data.cpu().numpy().all() >= 0.0, "Decay > 0"
        return batch_size, num_groups, T_plus_2, \
        all_cell, all_cell_bar, all_gate_decay, all_gate_output, \
        all_hidden, all_hidden_after_update

    def getTarget(self, event, dtime):
        r"""
        make target variable and masks
        """
        batch_size, num_groups, T_plus_2 = event.size()
        mask_complete = torch.ones_like(dtime[:, :, 1:])
        target_data = event[:, :, 1:].detach().data.clone()

        mask_complete[target_data >= self.event_num] = 0.0
        target_data[target_data >= self.event_num] = 0 # PAD to be 0
        target = target_data
        return target, mask_complete

    def getSampledStates(
        self, dtime_sampling, index_of_hidden_sampling,
        all_cell, all_cell_bar, all_gate_output, all_gate_decay):
        r"""
        we output the sampled hidden states of the left-to-right machine
        states shape : batch_size * num_groups * T+1 * hidden_dim
        dtime_sampling : batch_size * num_groups * max_len_sampling
        index_of_hidden_sampling : batch_size * num_groups * max_len_sampling
        """
        batch_size, num_groups, T_plus_1, _ = all_cell.size()   #256 1 9 16   ##256 1 200 16
        _, _, max_len_sampling = dtime_sampling.size()     #256  1 8  ##256 1 199
        # a=all_cell.view(
        #     batch_size * num_groups * T_plus_1, self.hidden_dim)
        # b=index_of_hidden_sampling.view(-1)
        # c=a[:2048,:].view(
        #             batch_size, num_groups, max_len_sampling, self.hidden_dim)
        #hidden_num=index_of_hidden_sampling.view(-1).size()
        hidden_num=256*max_len_sampling
        '''
        all_cell_sampling = all_cell.view(
            batch_size * num_groups * T_plus_1, self.hidden_dim )[
                index_of_hidden_sampling.view(-1), :].view(
                    batch_size, num_groups, max_len_sampling, self.hidden_dim)
    '''
        all_cell_sampling = all_cell.view(
            batch_size * num_groups * T_plus_1, self.hidden_dim)[
                            :hidden_num, :].view(
            batch_size, num_groups, max_len_sampling, self.hidden_dim)
        all_cell_bar_sampling = all_cell_bar.view(
            batch_size * num_groups * T_plus_1, self.hidden_dim )[
                :hidden_num, :].view(
                    batch_size, num_groups, max_len_sampling, self.hidden_dim)
        all_gate_output_sampling = all_gate_output.view(
            batch_size * num_groups * T_plus_1, self.hidden_dim )[
                :hidden_num, :].view(
                    batch_size, num_groups, max_len_sampling, self.hidden_dim)
        all_gate_decay_sampling = all_gate_decay.view(
            batch_size * num_groups * T_plus_1, self.hidden_dim )[
                :hidden_num, :].view(
                    batch_size, num_groups, max_len_sampling, self.hidden_dim)
        #执行公式7、4b
        cy_sample, hy_sample = self.rnn_cell.decay(
            all_cell_sampling, all_cell_bar_sampling,
            all_gate_decay_sampling, all_gate_output_sampling,
            dtime_sampling
        )

        return hy_sample

    def getLambda(
        self, batch_size, num_groups, T_plus_2,
        target, mask_complete, all_hidden, sampled_hidden ):
        #print("type is {}".format(type(all_hidden) ) )

        all_lambda= F.softplus(self.hidden_lambda(all_hidden), beta=self.beta)
        log_lambda= torch.log(all_lambda+ self.eps)

        #print("batchsize {}, num_groups {}, T_plus_2 {}".format(
        #    batch_size, num_groups, T_plus_2))

        log_lambda_target = log_lambda.view(
            batch_size * num_groups * (T_plus_2 - 1), self.event_num
        )[
            torch.arange(0, batch_size * num_groups * (T_plus_2 - 1),
                         dtype=torch.int64, device=self.device),
            target.view( batch_size * num_groups * (T_plus_2 - 1) )
        ].view(batch_size, num_groups, T_plus_2 - 1)

        log_lambda_target_complete = log_lambda_target * mask_complete

        lambda_sum_complete = torch.sum(all_lambda, dim=3)

        log_lambda_sum_complete = torch.log(lambda_sum_complete + self.eps)
        log_lambda_sum_complete *= mask_complete

        all_lambda_sample = F.softplus(
            self.hidden_lambda(sampled_hidden), beta=self.beta )

        return log_lambda_target_complete, all_lambda_sample

    def getIntegral(
        self, lambda_sample, mask_sampling, duration ):
        r"""
        mask_sampling : batch_size * num_groups * max_len_sampling
        duration : batch_size * num_groups
        """
        lambda_sample_sum = lambda_sample.sum(dim=3)
        lambda_sample_mean = torch.sum(
            lambda_sample_sum * mask_sampling, dim=2 ) / torch.sum(
            mask_sampling, dim=2 )
        integral = lambda_sample_mean * duration

        return integral

    def forward(self, input):

        event, feats, dtime, duration, \
        dtime_sampling, index_of_hidden_sampling, mask_sampling = input

        r"""
        event, dtime : batch_size, M, T+2
        post(erior of incomplete unobserved) : batch_size, M (not used)
        duration : batch_size, M
        dtime_sampling : batch_size, M, T_sample
        """
        #核心LSTM计算部分
        batch_size, num_groups, T_plus_2, \
        all_cell, all_cell_bar, all_gate_decay, all_gate_output, \
        all_hidden, all_hidden_after_update = self.getStates(event, dtime)

        assert num_groups == 1, "more than one group?"
        #遮罩
        target, mask_complete = self.getTarget( event, dtime )
        #公式4b
        sampled_hidden = self.getSampledStates(
            dtime_sampling, index_of_hidden_sampling,
            all_cell, all_cell_bar, all_gate_output, all_gate_decay
        )
        #公式4a
        # <s> \lambda_{k_i}(t_i | H_i) for events
        log_lambda_target_complete, all_lambda_sample = self.getLambda(
            batch_size, num_groups, T_plus_2,
            target, mask_complete, all_hidden, sampled_hidden )
        # batch_size * num_groups * T_plus_2-1
        # </s> \lambda_{k_i}(t_i) for events

        # <s> int_{0}^{T} lambda_sum dt for events
        integral_of_lambda_complete = self.getIntegral(
            all_lambda_sample, mask_sampling, duration )

        # batch_size * num_groups
        # </s> int_{0}^{T} lambda_sum dt for events

        # <s> log likelihood computation                                                                                `
        logP_complete = log_lambda_target_complete.sum(2) - integral_of_lambda_complete
        # log p --- according to (trained) neural Hawkes process

        # </s> log likelihood computation

        # complete log likelihood
        objective = -torch.sum( logP_complete )
        num_events = torch.sum( mask_complete )

        #return objective, num_events
        #计算时序预测值
        return objective,num_events,all_lambda_sample

max_stars_repo_name: LitteleStar/TDPP

max_issues_repo_name: LitteleStar/TDPP

max_forks_repo_name: LitteleStar/TDPP