text:
# Copyright 2017-2020 Lawrence Livermore National Security, LLC and other
# CallFlow Project Developers. See the top-level LICENSE file for details.
#
# SPDX-License-Identifier: MIT

import pandas as pd
import networkx as nx
import numpy as np
from ast import literal_eval as make_list

# CallFlow imports
try:
    import callflow
    from callflow.timer import Timer
    from callflow import SuperGraph

    LOGGER = callflow.get_logger(__name__)
except Exception:
    raise Exception("Module callflow not found not found.")

class SankeyLayout:
    """
    Sankey layout
    """

    _COLUMNS = [
        "actual_time",
        "time (inc)",
        "module",
        "name",
        "time",
        "type",
        "module",
        "entry_function",
    ]

    _PRIMARY_GROUPBY_COLUMN = "name"
    _SECONDARY_GROUPBY_COLUMN = "module"

    def __init__(
        self,
        supergraph,
        path="path",
        reveal_callsites=[],
        split_entry_module="",
        split_callee_module="",
    ):
        assert isinstance(supergraph, SuperGraph)
        assert isinstance(path, str)
        assert path in ["path", "group_path", "component_path"]

        # Set the current graph being rendered.
        self.supergraph = supergraph
        self.path = path

        self.timer = Timer()

        self.runs = self.supergraph.gf.df["dataset"].unique()

        self.reveal_callsites = reveal_callsites
        self.split_entry_module = split_entry_module
        self.split_callee_module = split_callee_module

        LOGGER.info(
            "Creating the Single SankeyLayout for {0}.".format(self.supergraph.tag)
        )

        self.primary_group_df = self.supergraph.gf.df.groupby(
            [SankeyLayout._PRIMARY_GROUPBY_COLUMN]
        )
        self.secondary_group_df = self.supergraph.gf.df.groupby(
            [SankeyLayout._SECONDARY_GROUPBY_COLUMN]
        )
        self.secondary_primary_group_df = self.supergraph.gf.df.groupby(
            [
                SankeyLayout._SECONDARY_GROUPBY_COLUMN,
                SankeyLayout._PRIMARY_GROUPBY_COLUMN,
            ]
        )

        with self.timer.phase("Construct Graph"):
            self.nxg = SankeyLayout._create_nxg_from_paths(
                self.supergraph.gf.df, self.path
            )
            self.add_reveal_paths(self.reveal_callsites)
            if self.split_entry_module != "":
                self.add_entry_callsite_paths(self.split_entry_module)
            if self.split_callee_module != "":
                self.add_exit_callees_paths()

        with self.timer.phase("Add graph attributes"):
            self._add_node_attributes()
            self._add_edge_attributes()

        LOGGER.debug(self.timer)

    # --------------------------------------------------------------------------
    # Split by reveal callsite.
    def create_source_targets(self, component_path):
        module = ""
        edges = []
        for idx, callsite in enumerate(component_path):
            if idx == 0:
                module = component_path[0]
                edges.append(
                    {
                        "module": module,
                        "source": module,
                        "target": module + "=" + component_path[idx + 1],
                    }
                )
                pass
            elif idx == len(component_path) - 1:
                pass
            else:
                edges.append(
                    {
                        "module": module,
                        "source": module + "=" + component_path[idx],
                        "target": module + "=" + component_path[idx + 1],
                    }
                )

        return edges

    def callsitePathInformation(self, callsites):
        paths = []
        for callsite in callsites:
            df = self.primary_group_df.get_group(callsite)
            paths.append(
                {
                    "group_path": make_list(df["group_path"].unique()[0]),
                    "path": make_list(df["path"].unique()[0]),
                    "component_path": make_list(df["component_path"].unique()[0]),
                }
            )
        return paths

    def add_reveal_paths(self, reveal_callsites):
        paths = self.callsitePathInformation(reveal_callsites)

        for path in paths:
            component_edges = self.create_source_targets(path["component_path"])
            for idx, edge in enumerate(component_edges):
                module = edge["module"]

                # format module +  '=' + callsite
                source = edge["source"]
                target = edge["target"]

                if not self.nxg.has_edge(source, target):
                    if idx == 0:
                        source_callsite = source
                        # source_df = self.secondary_group_df.get_group((module))
                        source_node_type = "super-node"
                    else:
                        source_callsite = source.split("=")[1]
                        # source_df = self.secondary_primary_group_df.get_group(
                        #     (module, source_callsite)
                        # )
                        source_node_type = "component-node"

                    target_callsite = target.split("=")[1]
                    target_df = self.secondary_primary_group_df.get_group(
                        (module, target_callsite)
                    )
                    target_node_type = "component-node"

                    # source_weight = source_df["time (inc)"].max()
                    target_weight = target_df["time (inc)"].mean()

                    print(f"Adding edge: {source_callsite}, {target_callsite}")
                    self.nxg.add_node(source, attr_dict={"type": source_node_type})
                    self.nxg.add_node(target, attr_dict={"type": target_node_type})
                    self.nxg.add_edge(
                        source,
                        target,
                        attr_dict=[
                            {
                                "source_callsite": source_callsite,
                                "target_callsite": target_callsite,
                                "weight": target_weight,
                                "edge_type": "reveal_edge",
                            }
                        ],
                    )

    # --------------------------------------------------------------------------
    # Add callsites based on split by entry function interactions.
    def module_entry_functions_map(self, graph):
        entry_functions = {}
        for edge in graph.edges(data=True):
            attr_dict = edge[2]["attr_dict"]
            edge_tuple = (edge[0], edge[1])
            for edge_attr in attr_dict:
                if edge_tuple[1] not in entry_functions:
                    entry_functions[edge_tuple[1]] = []
                entry_functions[edge_tuple[1]].append(edge_attr["target_callsite"])
        return entry_functions

    def create_source_targets_from_group_path(self, path):
        edges = []
        for idx, callsite in enumerate(path):
            if idx == len(path) - 1:
                break
            source = path[idx].split("=")
            target = path[idx + 1].split("=")
            edges.append(
                {
                    "source": source[0],
                    "target": target[0],
                    "source_callsite": source[1],
                    "target_callsite": target[1],
                }
            )
        return edges

    def same_source_edges(self, component_edges, reveal_module):
        ret = []
        for idx, edge in enumerate(component_edges):
            source = edge["source"]

            if source == reveal_module:
                ret.append(edge)
        return ret

    def same_target_edges(self, component_edges, reveal_module):
        ret = []
        for idx, edge in enumerate(component_edges):
            target = edge["target"]

            if target == reveal_module:
                ret.append(edge)
        return ret

    def add_entry_callsite_paths(self, reveal_module):
        entry_functions_map = self.module_entry_functions_map(self.nxg)
        reveal_callsites = entry_functions_map[reveal_module]
        paths = self.callsitePathInformation(reveal_callsites)

        for path in paths:
            component_edges = self.create_source_targets_from_group_path(
                path["group_path"]
            )
            source_edges_to_remove = self.same_source_edges(
                component_edges, reveal_module
            )
            target_edges_to_remove = self.same_target_edges(
                component_edges, reveal_module
            )

            if len(source_edges_to_remove) != 0:
                for edge in source_edges_to_remove:
                    if self.nxg.has_edge(edge["source"], edge["target"]):
                        self.nxg.remove_edge((edge["source"], edge["target"]))
                    self.nxg.add_node(
                        reveal_module + "=" + edge["source_callsite"],
                        attr_dict={"type": "component-node"},
                    )
                    self.nxg.add_edge(
                        (reveal_module + "=" + edge["source_callsite"], edge["target"]),
                        attr_dict=[
                            {
                                "source_callsite": edge["source_callsite"],
                                "target_callsite": edge["target_callsite"],
                                "weight": self.module_name_group_df.get_group(
                                    (reveal_module, edge["source_callsite"])
                                )["time (inc)"].max(),
                                "edge_type": "reveal_edge",
                            }
                        ],
                    )

            if len(target_edges_to_remove) != 0:
                for edge in target_edges_to_remove:
                    if self.nxg.has_edge(edge["source"], edge["target"]):
                        self.nxg.remove_edge(edge["source"], edge["target"])
                    self.nxg.add_node(
                        reveal_module + "=" + edge["target_callsite"],
                        attr_dict={"type": "component-node"},
                    )
                    self.nxg.add_edge(
                        edge["source"],
                        reveal_module + "=" + edge["target_callsite"],
                        attr_dict=[
                            {
                                "source_callsite": edge["source_callsite"],
                                "target_callsite": edge["target_callsite"],
                                "weight": self.secondary_primary_group_df.get_group(
                                    (edge["target"], edge["target_callsite"])
                                )["time (inc)"].max(),
                                "edge_type": "reveal_edge",
                            }
                        ],
                    )

        self.nxg.remove_node(reveal_module)

    # --------------------------------------------------------------------------
    # Node attribute methods.
    def _add_node_attributes(self):
        """
        Adds node attributes from the dataframe using the _COLUMNS.
        """
        ensemble_mapping = SankeyLayout._ensemble_map(
            df=self.supergraph.gf.df, nxg=self.nxg, columns=SankeyLayout._COLUMNS
        )
        for idx, key in enumerate(ensemble_mapping):
            nx.set_node_attributes(self.nxg, name=key, values=ensemble_mapping[key])

        dataset_mapping = {}
        for run in self.runs:
            dataset_mapping[run] = SankeyLayout._dataset_map(
                df=self.supergraph.gf.df,
                nxg=self.nxg,
                tag=run,
                columns=SankeyLayout._COLUMNS,
            )
            nx.set_node_attributes(
                self.nxg, name=self.supergraph.tag, values=dataset_mapping[run]
            )

    # --------------------------------------------------------------------------
    # Edge attribute methods.
    def _add_edge_attributes(self):
        edge_type_mapping = SankeyLayout.edge_type(self.nxg)
        nx.set_edge_attributes(self.nxg, name="edge_type", values=edge_type_mapping)

        inclusive_flow = SankeyLayout.edge_weight(self.nxg)
        nx.set_edge_attributes(self.nxg, name="weight", values=inclusive_flow)

        entry_functions_mapping = SankeyLayout.entry_functions(self.nxg)
        nx.set_edge_attributes(
            self.nxg, name="entry_callsites", values=entry_functions_mapping
        )

        exit_functions_mapping = SankeyLayout.exit_functions(self.nxg)
        nx.set_edge_attributes(
            self.nxg, name="exit_callsites", values=exit_functions_mapping
        )

    # --------------------------------------------------------------------------
    @staticmethod
    def module_time(group_df, module_callsite_map, module):
        """
        For node attributes: Calculates the time spent inside the module overall
        """
        exc_time_sum = 0
        inc_time_max = 0
        for callsite in module_callsite_map[module]:
            callsite_df = group_df.get_group((module, callsite))
            max_inc_time = callsite_df["time (inc)"].mean()
            inc_time_max = max(inc_time_max, max_inc_time)
            max_exc_time = callsite_df["time"].max()
            exc_time_sum += max_exc_time
        return {"Inclusive": inc_time_max, "Exclusive": exc_time_sum}

    @staticmethod
    def callsite_time(group_df, module, callsite):
        """
        For node attribute: Calculates the time spent by each callsite.
        """
        callsite_df = group_df.get_group((module, callsite))
        max_inc_time = callsite_df["time (inc)"].mean()
        max_exc_time = callsite_df["time"].mean()

        return {"Inclusive": max_inc_time, "Exclusive": max_exc_time}

    # --------------------------------------------------------------------------
    # Construct the networkX Graph from call paths.
    @staticmethod
    def _create_nxg_from_paths(df, path):
        """
        Construct a networkx graph from paths.
        Note: Current logic constructs two graphs (one for cct, and one for supergraph) and later uses them to construct a module level supergraph.
        """
        assert isinstance(df, pd.DataFrame)
        assert path in df.columns
        assert "name" in df.columns
        assert "module" in df.columns

        # Get the grouped dataframes.
        paths_df = df.groupby(["name", path])
        module_name_group_df = df.groupby(["module", "name"])

        # Empty networkx graphs.
        nxg = nx.DiGraph()
        cct = nx.DiGraph()

        for (callsite, path), path_df in paths_df:
            # Break cycles, if any.
            path_list = SankeyLayout._break_cycles_in_paths(path)

            # loop through the path lists for each callsite.
            for callsite_idx, callsite in enumerate(path_list):
                if callsite_idx != len(path_list) - 1:
                    source = path_list[callsite_idx]
                    target = path_list[callsite_idx + 1]

                    source_df = module_name_group_df.get_group(
                        (source["module"], source["callsite"])
                    )
                    target_df = module_name_group_df.get_group(
                        (target["module"], target["callsite"])
                    )

                    has_caller_edge = nxg.has_edge(source["module"], target["module"])
                    has_callback_edge = nxg.has_edge(target["module"], source["module"])
                    has_cct_edge = cct.has_edge(source["callsite"], target["callsite"])

                    if has_callback_edge:
                        edge_type = "callback"
                    else:
                        edge_type = "caller"

                    edge_dict = {
                        "source_callsite": source["callsite"],
                        "target_callsite": target["callsite"],
                        "edge_type": edge_type,
                        "weight": target_df["time (inc)"].mean(),
                        "source_dataset": source_df["dataset"].unique().tolist(),
                        "target_dataset": target_df["dataset"].unique().tolist(),
                    }

                    node_dict = {"type": "super-node"}

                    # If the module-module edge does not exist.
                    if (
                        not has_caller_edge
                        and not has_cct_edge
                        and not has_callback_edge
                    ):
                        LOGGER.info(
                            f"Add {edge_type} edge for : {source['module']}--{target['module']}"
                        )
                        nxg.add_node(source["module"], attr_dict=node_dict)
                        nxg.add_node(target["module"], attr_dict=node_dict)
                        nxg.add_edge(
                            source["module"], target["module"], attr_dict=[edge_dict]
                        )

                    # Edge exists for source["module"] -> target["module"]
                    elif not has_cct_edge and not has_callback_edge:
                        # edge_data = nxg.get_edge_data(
                        #     *(source["module"], target["module"])
                        # )
                        nxg[source["module"]][target["module"]]["attr_dict"].append(
                            edge_dict
                        )

                    # If edge is not in CCT. Add it.
                    if not has_cct_edge:
                        cct.add_edge(
                            source["callsite"],
                            target["callsite"],
                            attr_dict={"weight": target_df["time (inc)"].mean()},
                        )

        return nxg

    @staticmethod
    def _break_cycles_in_paths(path):
        """
        Breaks cycles if present in the callpath.

        Parameter:
            path: path array
        """
        from ast import literal_eval as make_list

        ret = []
        moduleMapper = {}
        dataMap = {}

        # TODO: see if we can remove this.
        if isinstance(path, float):
            return []

        path_list = make_list(path)

        for idx, elem in enumerate(path_list):
            callsite = elem.split("=")[1]
            module = elem.split("=")[0]
            if module not in dataMap:
                moduleMapper[module] = 0
                dataMap[module] = [
                    {"callsite": callsite, "module": module, "level": idx}
                ]
            else:
                flag = [p["level"] == idx for p in dataMap[module]]
                if np.any(np.array(flag)):
                    moduleMapper[module] += 1
                    dataMap[module].append(
                        {
                            "callsite": callsite,
                            "module": module + "=" + callsite,
                            "level": idx,
                        }
                    )
                else:
                    dataMap[module].append(
                        {"callsite": callsite, "module": module, "level": idx}
                    )
            ret.append(dataMap[module][-1])

        return ret

    @staticmethod
    def _ensemble_map(df, nxg, columns=[]):
        """
        Creates maps for all columns for the ensemble data.
        Note: For single supergraph, ensemble_map would be the same as dataset_map[single_run.tag].
        TODO: There is code repetition among ensemble_maps and dataset_maps. Need to simplify.
        """
        assert isinstance(df, pd.DataFrame)
        assert isinstance(nxg, nx.DiGraph)
        ret = {}

        module_group_df = df.groupby(["module"])
        module_name_group_df = df.groupby(["module", "name"])

        module_callsite_map = module_group_df["name"].unique().to_dict()

        module_time_inc_map = module_group_df["time (inc)"].max().to_dict()
        module_time_exc_map = module_group_df["time"].max().to_dict()

        name_time_inc_map = module_name_group_df["time (inc)"].max().to_dict()
        name_time_exc_map = module_name_group_df["time"].max().to_dict()

        # loop through the nodes
        for node in nxg.nodes(data=True):
            node_name = node[0]
            node_dict = node[1]["attr_dict"]

            if node_dict["type"] == "component-node":
                module = node_name.split("=")[0]
                callsite = node_name.split("=")[1]
                actual_time = SankeyLayout.callsite_time(
                    group_df=module_name_group_df, module=module, callsite=callsite
                )
                time_inc = name_time_inc_map[(module, callsite)]
                time_exc = name_time_exc_map[(module, callsite)]

            elif node_dict["type"] == "super-node":
                module = node_name
                callsite = module_callsite_map[module].tolist()
                actual_time = SankeyLayout.module_time(
                    group_df=module_name_group_df,
                    module_callsite_map=module_callsite_map,
                    module=module,
                )

                time_inc = module_time_inc_map[module]
                time_exc = module_time_exc_map[module]

            for column in columns:
                if column not in ret:
                    ret[column] = {}

                if column == "time (inc)":
                    ret[column][node_name] = time_inc

                elif column == "time":
                    ret[column][node_name] = time_exc

                elif column == "actual_time":
                    ret[column][node_name] = actual_time

                elif column == "module":
                    ret[column][node_name] = module

                elif column == "name":
                    ret[column][node_name] = callsite

                elif column == "type":
                    ret[column][node_name] = node_dict["type"]

        return ret

    @staticmethod
    def _dataset_map(df, nxg, columns=[], tag=""):
        """
        Creates maps for all node attributes (i.e., columns in df) for each dataset.
        """
        assert isinstance(df, pd.DataFrame)
        assert isinstance(nxg, nx.DiGraph)

        ret = {}

        # Reduce the entire_df to respective target dfs.
        target_df = df.loc[df["dataset"] == tag]

        # Unique modules in the target run
        # target_modules = target_df["module"].unique()

        # Group the dataframe in two ways.
        # 1. by module
        # 2. by module and callsite
        target_module_group_df = target_df.groupby(["module"])
        target_module_name_group_df = target_df.groupby(["module", "name"])

        # Module map for target run {'module': [Array of callsites]}
        target_module_callsite_map = target_module_group_df["name"].unique().to_dict()

        # Inclusive time maps for the module level and callsite level.
        target_module_time_inc_map = (
            target_module_group_df["time (inc)"].max().to_dict()
        )
        target_name_time_inc_map = (
            target_module_name_group_df["time (inc)"].max().to_dict()
        )

        # Exclusive time maps for the module level and callsite level.
        target_module_time_exc_map = target_module_group_df["time"].max().to_dict()
        target_name_time_exc_map = target_module_name_group_df["time"].max().to_dict()

        for node in nxg.nodes(data=True):
            node_name = node[0]
            node_dict = node[1]["attr_dict"]
            if node_name in target_module_callsite_map.keys():
                if node_dict["type"] == "component-node":
                    module = node_name.split("=")[0]
                    callsite = node_name.split("=")[1]
                    agg_time = SankeyLayout.callsite_time(
                        group_df=target_module_group_df,
                        module=module,
                        callsite=callsite,
                    )
                    time_inc = target_name_time_inc_map[(module, callsite)]
                    time_exc = target_name_time_exc_map[(module, callsite)]

                elif node_dict["type"] == "super-node":
                    module = node_name
                    callsite = target_module_callsite_map[module].tolist()
                    agg_time = SankeyLayout.module_time(
                        group_df=target_module_name_group_df,
                        module_callsite_map=target_module_callsite_map,
                        module=module,
                    )

                    time_inc = target_module_time_inc_map[module]
                    time_exc = target_module_time_exc_map[module]

                if node_name not in ret:
                    ret[node_name] = {}

                for column in columns:
                    if column == "time (inc)":
                        ret[node_name][column] = time_inc

                    elif column == "time":
                        ret[node_name][column] = time_exc

                    elif column == "module":
                        ret[node_name][column] = module

                    elif column == "actual_time":
                        ret[node_name][column] = agg_time

                    elif column == "name":
                        ret[node_name][column] = callsite

                    elif column == "type":
                        ret[node_name][column] = node_dict["type"]

                    elif column == "entry_function":
                        ret[node_name][column] = SankeyLayout.get_entry_functions(
                            target_module_group_df, node_name
                        )

        return ret

    @staticmethod
    def get_entry_functions(df, module):
        """
        Get the entry function of a module from the dataframe.
        """
        module_df = df.get_group(module)
        entry_func_df = module_df.loc[module_df["entry_function"]]
        return entry_func_df["callees"].unique().tolist()[0]

    # --------------------------------------------------------------------------
    @staticmethod
    def edge_type(nxg):
        """
        For edge attribute: Appends the edge type ("caller", or "callback")
        """
        ret = {}
        for edge in nxg.edges(data=True):
            ret[(edge[0], edge[1])] = edge[2]["attr_dict"][0]["edge_type"]
        return ret

    @staticmethod
    def edge_weight(nxg):
        """
        For edge attribute: Calculates the sankey flow between nodes.
        """
        flow_mapping = {}
        for edge in nxg.edges(data=True):
            if (edge[0], edge[1]) not in flow_mapping:
                flow_mapping[(edge[0], edge[1])] = 0

            attr_dict = edge[2]["attr_dict"]
            for d in attr_dict:
                flow_mapping[(edge[0], edge[1])] += d["weight"]

        ret = {}
        for edge in nxg.edges(data=True):
            edge_tuple = (edge[0], edge[1])
            if edge_tuple not in flow_mapping:
                # Check if it s a reveal edge
                attr_dict = edge[2]["attr_dict"]
                if attr_dict["edge_type"] == "reveal_edge":
                    flow_mapping[edge_tuple] = attr_dict["weight"]
                    ret[edge_tuple] = flow_mapping[edge_tuple]
                else:
                    ret[edge_tuple] = 0
            else:
                ret[edge_tuple] = flow_mapping[edge_tuple]

        return ret

    @staticmethod
    def entry_functions(nxg):
        """
        For edge attribute: Adds the entry functions for the module.
        """
        entry_functions = {}
        for edge in nxg.edges(data=True):
            attr_dict = edge[2]["attr_dict"]
            edge_tuple = (edge[0], edge[1])
            for edge_attr in attr_dict:
                if edge_tuple not in entry_functions:
                    entry_functions[edge_tuple] = []
                entry_functions[edge_tuple].append(edge_attr["target_callsite"])
        return entry_functions

    @staticmethod
    def exit_functions(nxg):
        """
        For edge attribute: Adds the exit functions from the module.
        """
        exit_functions = {}
        for edge in nxg.edges(data=True):
            attr_dict = edge[2]["attr_dict"]
            edge_tuple = (edge[0], edge[1])
            for edge_attr in attr_dict:
                if edge_tuple not in exit_functions:
                    exit_functions[edge_tuple] = []
                exit_functions[edge_tuple].append(edge_attr["source_callsite"])
        return exit_functions

max_stars_repo_name: jarusified/CallFlow

max_issues_repo_name: jarusified/CallFlow

max_forks_repo_name: jarusified/CallFlow

reason: import numpy,import networkx


text:
import numpy as np
import csv
import ast
import pickle

"""
Neighborhood Network package
Simple and weighted network objects with built-in neighborhood list for scalable random walk and MCMC sampling
Author: Josh Vendrow and Hanbaek Lyu
"""

class NNetwork():

    def __init__(self):
        self.edges = []
        # self.sorted_left = None
        # self.sorted_right = None
        self.neighb = {}
        self.vertices = []
        self.vertices_set = set()
        self.number_nodes = 0

    """
    def set_edges(self, edges):
        self.sort_left(edges)
        self.sort_right(edges)
        self.edges = edges
        self.set_neighbors()
    """

    def add_edges(self, edges):
        """Given an edgelist, add each edge in the edgelist to the network"""
        i = 0
        for edge in edges:
            self.add_edge(edge, update=False)
            # print('Loading %i th edge out of %i edges' % (i, len(edges)))
            # i += 1

        # self.node = list(self.neighb.keys())

    def add_edge(self, edge, update=True):
        """Given an edge, add this edge to the Network"""

        u, v = edge
        self.edges.append(edge)
        if u not in self.neighb:
            self.neighb[u] = set({v})
            self.vertices.append(u)
            self.vertices_set.add(u)
            self.number_nodes += 1

        else:
            self.neighb[u].add(v)

        if v not in self.neighb:
            self.neighb[v] = set({u})
            self.vertices.append(v)
            self.vertices_set.add(v)
            self.number_nodes += 1

        else:
            self.neighb[v].add(u)

    def add_nodes(self, nodes):
        """Given a list of nodes, adds the nodes to the Network"""

        for node in nodes:
            self.add_node(node)

    def add_node(self, node):
        """Given a single node, adds the node to the Network"""

        if node not in self.vertices:
            self.vertices.append(node)
            self.neighb[node] = set()
            self.number_nodes += 1
            self.vertices_set.add(node)

    def set_neighbors(self):

        for edge in self.edges:
            self.add_edge(edge)

    def get_edges(self):
        # this may create unecessary large lists
        set_edgelist = []
        for x in self.vertices:
            if x in self.neighb:
                for y in self.neighb[x]:
                    set_edgelist.append([x, y])
        self.edges = set_edgelist
        return self.edges

        # self.node = list(self.neighb.keys())

    def intersection(self, Network_other):
        """
        Given another network, returns all edges found in both networks.
        """
        common_nodeset = self.vertices_set.intersection(Network_other.vertices_set)
        common_edgelist = []

        for x in common_nodeset:
            for y in self.neighb[x].intersection(Network_other.neighbors(x)):
                common_edgelist.append([x, y])

        return common_edgelist

    def neighbors(self, node):
        """
        Given a node, returns all the neighbors of the node.
        """
        if node not in self.nodes():
            print('ERROR: %s not in the node set' % node)
        return self.neighb[node]

    def has_edge(self, u, v):
        """
        Given two nodes, returns true of these is an edge between then, false otherwise.
        """
        try:
            return v in self.neighb[u]
        except KeyError:
            return False

    def nodes(self, is_set=False):
        return self.vertices if not is_set else self.vertices_set

    def num_nodes(self):
        return self.number_nodes

    def edges(self):
        return self.edges

class Wtd_NNetwork():
    '''
    Class for weighted and undirectional network
    Can store additional tensor information in colored_edge_weights -- list of nonnegative numbers
    Underlying NNetwork serves as the skeleton for RWs and motif sampling
    colored edges = additional information on the (already weighted) edges
    Think of edges = pixel location and colored edges = color of that pixel
    '''

    def __init__(self):
        # self.edges = []
        self.wtd_edges = {}
        self.neighb = {}
        self.vertices = []
        self.vertices_set = set()
        self.number_nodes = 0
        self.colored_edge_weights = {}
        self.color_dim = 0

    def add_edges(self, edges, edge_weight=1, increment_weights=True):
        """Given an edgelist, add each edge in the edgelist to the network"""
        for edge in edges:
            self.add_edge(edge, weight=edge_weight, increment_weights=increment_weights)
            # self.edge_weights.update({str(edge): default_edge_weight})

    def add_edge(self, edge, weight=float(1), increment_weights=False, is_dict=False):
        """Given an edge, add this edge to the Network"""
        # if is_dict, then edge is the form of "['123', '456']".
        if not is_dict:
            u, v = edge
        else:
            u, v = eval(edge)

        u = str(u)
        v = str(v)
        # self.edges.append(edge)
        if not increment_weights or not self.has_edge(u, v):
            wt = weight
        else:
            wt = self.get_edge_weight(u, v) + weight

        self.wtd_edges.update({str([str(u), str(v)]): wt})
        self.wtd_edges.update({str([str(v), str(u)]): wt})

        if u not in self.neighb:
            self.neighb[u] = {v}
            self.vertices.append(u)
            self.vertices_set.add(u)
            self.number_nodes += 1
        else:
            self.neighb[u].add(v)

        if v not in self.neighb:
            self.neighb[v] = {u}
            self.vertices.append(v)
            self.vertices_set.add(v)
            self.number_nodes += 1
        else:
            self.neighb[v].add(u)

    def add_nodes(self, nodes):
        """Given a list of nodes, adds the nodes to the Network"""
        for node in nodes:
            self.add_node(node)

    def add_node(self, node):
        """Given a single node, adds the node to the Network"""
        if node not in self.vertices:
            self.vertices.append(node)
            self.neighb[node] = set()
            self.number_nodes += 1
            self.vertices_set.add(node)

    def set_neighbors(self):
        for edge in self.edges:
            self.add_edge(edge)

    def uniq(self, lst):
        ### get rid of duplicates in an input list "lst"
        last = object()
        for item in lst:
            if item == last:
                continue
            yield item
            last = item
        ### Returns a generator object: take "list" to turn the output into a list

    def intersection(self, Network_other):
        """
        Given another network, returns all edges found in both networks.
        """
        common_nodeset = self.vertices_set.intersection(Network_other.vertices_set)
        common_edgelist = []

        for x in common_nodeset:
            for y in self.neighb[x].intersection(Network_other.neighbors(x)):
                common_edgelist.append([x, y])

        return common_edgelist

    def intersection_slow(self, Network_other):
        """
        Given another network, returns all edges found in both networks.
        """
        edges_other = list(self.uniq(Network_other.get_edges()))
        print('edges_other', edges_other)
        edges = list(self.uniq(self.get_edges()))
        print('edges', edges)
        intersection = [e for e in edges if e in edges_other]
        print('intersection', intersection)
        return intersection

    def neighbors(self, node):
        """
        Given a node, returns all the neighbors of the node.
        """
        if node not in self.nodes():
            print('ERROR: node %s not in the node set' % str(node))

        return self.neighb[node]

    def has_edge(self, u, v):
        """
        Given two nodes, returns true of these is an edge between then, false otherwise.
        """
        try:
            return v in self.neighb[u]
        except KeyError:
            return False

    def indices(self, a, func):
        return [i for (i, val) in enumerate(a) if func(val)]

    def read_adj(self, adj):
        """
        adj = n x n (weighted) adjacency matrix
        """
        for i in np.arange(adj.shape[0]):
            nb_list = self.indices(adj[:, i], lambda x: x == 1)
            for j in nb_list:
                self.add_edge(edge=[i,j], weight=float(adj[i,j]), increment_weights=False, is_dict=False)

    def has_colored_edge(self, u, v):
        """
        Given two nodes, returns true of these is a colored edge between then, false otherwise.
        """
        colored_edge = self.get_colored_edge_weight(u, v)
        return colored_edge != None

    def nodes(self, is_set=False):
        return self.vertices if not is_set else self.vertices_set

    def num_nodes(self):
        return self.number_nodes

    def get_edge_weight(self, u, v):
        return self.wtd_edges.get(str([str(u), str(v)]))

    def get_edges(self):
        set_edgelist = []
        for x in self.vertices:
            if x in self.neighb:  ### x has a neighbor
                for y in self.neighb[x]:
                    set_edgelist.append([x, y])
        self.edges = set_edgelist
        return self.edges

    def get_wtd_edgelist(self):
        edgelist = self.get_edges()
        wtd_edgelist = []
        for edge in edgelist:
            u, v = edge
            wtd_edgelist.append([u, v, abs(self.get_edge_weight(u, v))])

        return wtd_edgelist

    def get_abs_wtd_edgelist(self):
        edgelist = self.get_edges()
        wtd_edgelist = []
        for edge in edgelist:
            u, v = edge
            wtd_edgelist.append([u, v, self.get_edge_weight(u, v)])

        return wtd_edgelist

    def save_wtd_edgelist(self, default_folder='Temp_save_graphs', default_name='temp_wtd_edgelist'):
        edgelist = self.get_edges()
        wtd_edgelist = []
        for edge in edgelist:
            u, v = edge
            wtd_edgelist.append([u, v, self.get_edge_weight(u, v)])

        ### Todo: Use pickle later
        with open(default_folder + '/' + default_name + '.txt', "w") as file:
            file.write(str(wtd_edgelist))
        return wtd_edgelist

    def save_wtd_edges(self, path):
        # Does not creat additional list file and saves directly self.wtd_edges as a np dictionary.
        with open(path, 'wb') as handle:
            pickle.dump(self.wtd_edges, handle, protocol=pickle.HIGHEST_PROTOCOL)

    def add_wtd_edges(self, edges, increment_weights=False, is_dict=False):
        """Given an edgelist, add each edge in the edgelist to the network"""
        """if is dict, edges is a dictionary {edge: weight}"""
        if not is_dict:
            for wtd_edge in edges:
                if len(wtd_edge) == 2:
                    self.add_edge(wtd_edge[0:2], weight=1, increment_weights=increment_weights)
                else:
                    self.add_edge(wtd_edge[0:2], weight=float(wtd_edge[-1]), increment_weights=increment_weights)
            # self.get_edges()
        else:
            for edge in edges.keys():
                self.add_edge(edge, weight=edges.get(edge), increment_weights=increment_weights, is_dict=True)

    def load_add_wtd_edges(self, path, delimiter=',', increment_weights=False, use_genfromtxt=False, is_dict=False,
                           is_pickle=False):

        if is_dict and not is_pickle:
            wtd_edges = np.load(path, allow_pickle=True).items()
        if is_pickle:
            with open(path, 'rb') as handle:
                wtd_edges = pickle.load(handle)

        elif not use_genfromtxt:
            with open(path, "r") as file:
                wtd_edges = eval(file.readline())
        elif delimiter == '\t':
            with open(path) as f:
                reader = csv.reader(f, delimiter="\t")
                wtd_edges = list(reader)

        else:
            wtd_edges = np.genfromtxt(path, delimiter=delimiter, dtype=str)
            wtd_edges = wtd_edges.tolist()

        self.add_wtd_edges(edges=wtd_edges, increment_weights=increment_weights, is_dict=is_dict)

    def get_min_max_edge_weights(self):
        list_wts = []
        for edge in self.get_wtd_edgelist():
            list_wts.append(edge[-1])
        print('minimum edge weight:', min(list_wts))
        print('maximum edge weight:', max(list_wts))

    def clear_edges(self):
        node_set = self.vertices
        self.__init__()
        self.add_nodes(node_set)

    def threshold2simple(self, threshold=0.5):
        ### Threshold edges of the given weighted network and return a simple graph
        G_simple = NNetwork()
        for edge in self.get_edges():
            u, v = edge
            edge_weight = self.get_edge_weight(u, v)
            if edge_weight > threshold:
                G_simple.add_edge([u, v])
        return G_simple

    def add_colored_edges(self, colored_edges):
        """Given a list of colored edges, add each colored edge to the network"""
        for edge in colored_edges:
            self.add_colored_edge(edge)

    def add_colored_edge(self, colored_edge):
        """Given a colored edge, [u, v, a1, a2, ...], add this to the Network"""
        u, v = colored_edge[0:2]

        self.colored_edge_weights.update({str([str(u), str(v)]): colored_edge[2:]})
        self.colored_edge_weights.update({str([str(v), str(u)]): colored_edge[2:]})

    def set_clrd_edges_signs(self):
        """
        colored edge weight of and edge with weight w = [+(w), -(w)] (split positive and negative parts as a vector)
        """
        edgelist = self.get_edges()

        for edge in edgelist:
            u, v = edge
            w = self.get_edge_weight(u, v)
            colored_edge = [u, v, w * (w >= 0), -w * (w < 0)]
            self.add_colored_edge(colored_edge)

        u, v = self.get_edges()[0]
        self.color_dim = len(self.get_colored_edge_weight(u, v))

    def get_colored_edge_weight(self, u, v):
        return self.colored_edge_weights.get(str([str(u), str(v)]))

    def get_adjacency_matrix(self):
        mx = np.zeros(shape=(len(self.vertices), len(self.vertices)))
        for i in np.arange(len(self.vertices)):
            for j in np.arange(len(self.vertices)):
                if get_edge_weight(self.vertices[i], self.vertices[j]) > 0:
                    mx[i, j] = self.get_edge_weight(self.vertices[i], self.vertices[j])
        return mx

    def subgraph(self, nodelist):
        ### Take induced subgraph on the specified nodeset
        V0 = set(nodelist).intersection(self.vertices)
        G_sub = Wtd_NNetwork()
        for u in V0:
            nbh_sub = self.neighbors(u).intersection(set(nodelist))
            if len(nbh_sub) > 0:
                for v in nbh_sub:
                    G_sub.add_edge([u, v], weight=self.get_edge_weight(u, v))

        return G_sub

    def r_neighborhood(self, nodelist, radius=1):
        ### Take the induced subgraph of the r-neighborhood of a given subset of nodes
        V = set(nodelist).intersection(self.vertices)
        for r in np.arange(radius):
            for u in V:
                V = V.union(self.neighbors(u))

        return self.subgraph(V)

    def count_k_step_walks(self, u, radius=1):
        ### Counts the number of k-step walks starting from u
        ### Take the r-neighborhood, take the induced adjacency matrix Ar, and take r th power, and then sum
        G_r_nbh = self.r_neighborhood([u], radius=radius)
        vertices = G_r_nbh.vertices
        idx = vertices.index(u)

        Ar = np.zeros(shape=(len(G_r_nbh.vertices), len(G_r_nbh.vertices)))
        for i in np.arange(len(self.vertices)):
            for j in np.arange(len(self.vertices)):
                if G_r_nbh.has_edge(self.vertices[i], self.vertices[j]):
                    Ar[i, j] = G_r_nbh.get_edge_weight(self.vertices[i], self.vertices[j])

        B = np.linalg.matrix_power(Ar, radius - 1)
        print('!!!! B', B)

        return sum(B[idx, :])

    def k_node_ind_subgraph(self, k, center=None):
        # Computes a random k-node induced subgraph
        # Initialized simple symmetric RW uniformly at "center" node,
        # and collects neighboring node until we get k distinct nodes
        # if center is None, then initial node is chosen uniformly at random
        # Once a set of k distinct nodes are collected, take the induced subgraph on it
        if k > len(self.vertices):
            print("cannot take i% distinct nodes from a graph with %i nodes" % (k, len(self.vertices)))

        V0 = []
        while (len(V0) < k): # RW may be initialized in a small component with <k nodes
        # -- restart if it doesn't end within k^2 steps
            i = 0
            V0 = []
            x = np.random.choice(self.vertices)
            if center is not None:
                x = center
            V0.append(x)
            while (len(V0) < k) and (i<k**2):
                x = np.random.choice(list(self.neighbors(x)))
                V0.append(x)
                V0 = list(set(V0))
                i += 1
            # now take subgraph on V0:
        return self.subgraph(V0)

ext: py

reason: import numpy


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
@author: Kerkyra Asvesta
In this PYTHON file I show the MCMC analysis to evaluate the likelihood of the parameters in question for
the  tilted ΛCDM model with the parameter a is fixed (t-Λ (a fixed))
"""

import multiprocessing as mproc
from multiprocessing import Pool
import numpy as np
from numpy import *
from scipy.integrate import quad
from math import sqrt
import matplotlib.pyplot as plt
from scipy import interpolate, linalg, optimize
from scipy.sparse.linalg import inv
from numpy.linalg import multi_dot
from astropy.constants import c
import emcee
from chainconsumer import ChainConsumer
from chainconsumer.diagnostic import Diagnostic
from matplotlib import rcParams as rcp

rcp['figure.figsize'] = [9, 6]
rcp['figure.dpi'] = 80
rcp['xtick.labelsize'] = 15
rcp['ytick.labelsize'] = 15
rcp['axes.formatter.useoffset'] = False
rcp['axes.linewidth'] = 1.5
rcp['axes.axisbelow'] = False
rcp['xtick.major.size'] = 8
rcp['xtick.minor.size'] = 4
rcp['xtick.labelsize'] = 15
rcp['legend.fontsize'] = 15
rcp['xtick.direction'] = 'in'
rcp['ytick.major.width'] = 2
rcp['ytick.minor.width'] = 2
rcp['font.size'] = 20

### some constants ###
c_km = (c.to('km/s').value)
H0 = 70.

####################################
#                                  #
#    import the Pantheon SNIa data #
#                                  #
####################################

# import redshifts (cosmological + heliocentric), apparent magnitudes, error of app.magnitudes, systematic errors
# Get the data from the github repository Pantheon of Dan Scolnic #
# https://github.com/dscolnic/Pantheon/blob/master/lcparam_full_long_zhel.txt #
data = np.loadtxt('/home/kerky/anaconda3/SN1A DATA/PANTHEON_DATA/Scolnic_data_updated.txt', usecols=[1,2,4,5])

### list with all the systematics as found in the PANTHEON catalog ###

# get the full systematics from the same repository #
#https://github.com/dscolnic/Pantheon/blob/master/sys_full_long.txt #
sys = np.genfromtxt('/home/kerky/anaconda3/SN1A DATA/PANTHEON_DATA/systematics.txt', skip_header=1)

### The list sn_names contains all the supernovae names in the PANTHEON catalog ###
sn_names = np.genfromtxt('/home/kerky/anaconda3/SN1A DATA/PANTHEON_DATA/Scolnic_data_updated.txt', usecols=[0],dtype='str')

z_cmb= np.array(data[:,0]) ## CMB redshift

z_hel = np.array(data[:,1]) ## heliocentric redshift

mb = np.array(data[:,2]) ## apparent magnitude

dmb = np.array(data[:,3]) ## error on mb

sqdmb = np.square(dmb) ## array with the square of the mb errors

### Construction of the full systematic covariance matrix ###
st = np.array(sys) ## put the systematics in an array
cov = np.reshape(st, (1048, 1048)) ## build the covariance matrix
er_stat = np.zeros((1048, 1048), float)
er_stat[np.diag_indices_from(er_stat)] = sqdmb
er_total = er_stat + cov
diag = np.array([np.diagonal(er_total)])
size = len(z_cmb)
print(size)
C_covn = np.linalg.inv([er_total]).reshape(1048, 1048)### the covariance matrix

def integrand(z) : ## substituting Omega_m = 0.3 and Omega_l = 0.7
    return np.sqrt(1/(0.3*((1+z)**3) + 0.7))

###  Deceleration parameter in the tilted model using the ΛCDM comoving distance &
# fixed value of the parameter a (a = 0.5) ( t-Λ (α fixed) )
def q_aparam(z, b):
    ratio = 1/(0.5+(b*(quad(lambda r:integrand(r), 0., z)[0]**3)))
    return 0.5*(1-ratio)

### Hubble parameter in the ( t-Λ (α fixed) ) model ###
def H_aparam(z, b):
    f = lambda x: (1+q_aparam(x, b))/(1+x)
    integ = quad(f, 0.0, z, epsabs=np.inf)[0]
    ex = np.exp(integ)
    return ex*(H0)

### luminosity distance in the ( t-Λ (α fixed) ) model ###
def dl_aparam(z, b):
    f = lambda y: (1/(H_aparam(y, b)))
    inte = np.real(quad(f, 0.,z, epsabs=np.inf)[0])
    return inte*c_km*(1+z)

### Hubble-free luminosity distance in the ( t-Λ (α fixed) ) model ###
def Dl_aparam(z, b):
    return (H0/c_km)*(dl_aparam(z, b))

### Construct the theoretical apparent magnitude of the ( t-Λ (α fixed) ) model ###
def m_atheparam(z, b, Mcal):
        #Mcal = M +42.38 - 5*np.log10(cosmo.h)
    return np.hstack([5*np.log10(Dl_aparam(z[i], b)) + Mcal for i in range(size)])

### construct the chi squared function for the ( t-Λ (α fixed) ) ###
# the tilted ( t-Λ (α fixed) ) model contains 2 free parameters, b, Mcal
def chi_aparam(aparams):
    b = aparams[0]
    Mcal = aparams[1]
    res1 = np.array([m_atheparam(z_cmb, b, Mcal) - mb]).reshape(1, 1048)
    res3 = np.array([m_atheparam(z_cmb, b, Mcal) - mb]).reshape(1048, )
    return np.sum(np.linalg.multi_dot([res1, C_covn, res3]))

#########################
#                       #
#    MCMC               #
#                       #
#########################

### define the prior ###
def lnpriorl(theta):
    b, Mcal = theta
    if  0.0 < b < 35 and 23. < Mcal < 24.0:
        return 0.0
    return -np.inf

### combine into full log-probability function
def lnprobl(theta):
    lp = lnpriorl(theta)
    if not np.isfinite(lp):
        return -np.inf

    return lp + (-0.5*chi_aparam(theta))

### define no of parameters and walkers
### ndim defines the number of dimensions (parameters)
ndiml, nwalkersl = 2, 100

# create an initialization for the walkers
initial = np.array([ 5.8, 23.810 ])
posl = [np.array(initial) + 0.0001*np.random.randn(ndiml) for i in range(nwalkersl)]
### run mcmc using multiprocessing
with Pool() as pool:
    samplerl = emcee.EnsembleSampler(nwalkersl, ndiml, lnprobl, pool=pool)
    resultl = samplerl.run_mcmc(posl, 2000, progress = True) # 2000 is the no of steps
### generate the flat chains and discard the burn-in steps

full_flat_samplesl = samplerl.get_chain(flat=True)
np.savetxt("fullchains_qla_LCDM_2000 ",full_flat_samplesl)
b_ac, M_cal_ac = samplerl.get_autocorr_time(quiet=True)
print("The autocorrelation length for b is {0} and M_cal is {1}".format(b_ac, M_cal_ac))
flat_samplesl = samplerl.get_chain(discard=20, flat=True, thin=int(max([b_ac, M_cal_ac])))
np.savetxt("flatchains_qla_LCDM",flat_samplesl)

### The flatchains from the MCMC analysis discarded the burn-in steps
flatchains_qla_LCDM = np.loadtxt("/home/kerky/anaconda3/test_q/resultsfromq(L)/q(l)_LCDM_1048/flatchains_qla_LCDM")

params = ["b", "Mcal"]
### plot the chains for each parameter
for i in range(2):
    plt.figure()
    plt.title(params[i])
    x = np.arange(len(flatchains_qla_LCDM[:,i]))
    plt.plot(x,flatchains_qla_LCDM[:,i])
    plt.show()

### USE CHAIN CONSUMER TO GENERATE PLOTS ###

params = ["b",r"${\cal{M}}$"]
cc = ChainConsumer()
c = cc.add_chain(flatchains_qla_LCDM[:,:], parameters=params)
c.configure(contour_labels="confidence", kde= True, statistics="max",summary = False, shade_alpha=0.9,tick_font_size=11, label_font_size=15, sigmas=[1, 2], linewidths=1.2, colors="#673AB7", sigma2d = False, shade =True, flip = False)

###  The maximum likelihood values for the t-Λ with a cst model can be found using the Analysis class of the Chain Consumer
BF_LCDMa = c.analysis.get_summary().values()
bf_b = list(BF_LCDMa)[0][1]
bf_mcal = list(BF_LCDMa)[1][1]
truth = [bf_b, bf_mcal]
fig = c.plotter.plot(figsize=2.0, extents=[[0, 16], [23.75, 23.875]], display=True, truth = truth)

max_stars_repo_name: lkazantzi/tilted-cosmology

max_issues_repo_name: lkazantzi/tilted-cosmology

max_forks_repo_name: lkazantzi/tilted-cosmology

reason: import numpy,from numpy,from scipy,from astropy


text:
import numpy as np

def diagonal_interpolation(data):
    #zeroidx = np.where(~data.any(axis=1))[0]

    if np.any(np.isnan(data) == True):
        bool_matrix = np.isnan(data, dtype = bool)
        zeroidx = np.where(np.all(bool_matrix == True, axis=1))[0]
    else:
        vmin = np.nanmin(data)
        zeroidx = np.where(np.all(data == vmin, axis=1))[0]

    if zeroidx.size!= 0:
        data[zeroidx,:] = np.nan
        data[:,zeroidx] = np.nan
        length = np.shape(data)[0]
        diag = data.diagonal(0)

        for i in range(0, length):
            diag = data.diagonal(i)
            nan_idx = np.argwhere(np.isnan(diag))
            if nan_idx.size != 0:
                x = np.arange(len(diag))
                x = np.delete(x, nan_idx)
                y = np.delete(diag, nan_idx)
                if x.size == 0:
                    data[nan_idx, nan_idx+i] = 0
                    data[nan_idx+i, nan_idx] = 0
                else:
                    val = np.interp(nan_idx, x, y)
                    data[nan_idx, nan_idx+i] = val
                    data[nan_idx+i, nan_idx] = data[nan_idx, nan_idx+i]
    return data

max_stars_repo_name: cosmoskaluga/optimalTAD

max_issues_repo_name: cosmoskaluga/optimalTAD

max_forks_repo_name: cosmoskaluga/optimalTAD


text:
from abc import ABC, abstractmethod
from typing import Any

import numpy as np
from eit_ai.train_utils.metadata import MetaData
from eit_ai.train_utils.dataset import AiDatasetHandler
from enum import Enum

from logging import getLogger
logger = getLogger(__name__)

MODEL_SUMMARY_FILENAME='model_summary'

################################################################################
# Custom Exeptions/Errors for Models
################################################################################
class WrongOptimizerError(Exception):
    """"""
class WrongLossError(Exception):
    """"""
class WrongMetricsError(Exception):
    """"""
class WrongLearnRateError(Exception):
    """"""
class ModelNotDefinedError(Exception):
    """"""
class ModelNotPreparedError(Exception):
    """"""
################################################################################
# Abstract Class for Models
################################################################################

class AiModelHandler(ABC):

    model=None
    name:str=''
    specific_var:dict={}

    def __init__(self, metadata:MetaData=None, build:bool=False) -> None:
        super().__init__()
        if build:
            self.build(metadata)

    def get_model(self):
        """Return the model object"""
        return self.model

    def get_name(self):
        """Return the name of the model"""
        return self.name

    def build(self, metadata:MetaData)->None:
        """Build the model, ready to train
        this pipeline is defined by three internal methods

        self._define_model(metadata=metadata)
        self._get_specific_var(metadata=metadata)
        self._prepare_model()

        Args:
            metadata (MetaData):
        """
        self._define_model(metadata=metadata)
        self._get_specific_var(metadata=metadata)
        self._prepare_model()

    @abstractmethod
    def _define_model(self, metadata:MetaData)-> None:
        """1. method called during building:
        here have should be the model layer structure defined and
        stored in set "self.model"

        Args:
            metadata (MetaData):
        """

    @abstractmethod
    def _get_specific_var(self, metadata:MetaData)-> None:
        """2. method called during building:
        Responsible of gathering, preparing, and setting the specific
        variables needed to prepare the model (eg. optimizer, )
        those variables are stored in the dict "specific_var", which can be set
        depending the needs of the model, eg. :
            specific_var['optimizer']
            specific_var['loss']
            specific_var['metrix']

        Args:
            metadata (MetaData):

        Raises:
            WrongLossError: raised if passed metadata.loss is not in KERAS_LOSS list
            WrongOptimizerError: raised if passed metadata.optimizer is not in KERAS_OPTIMIZER list
            WrongMetrixError: raised if passed metadata.metrics is not a list #Could be better tested... TODO
            WrongLearnRateError: raised if passed metadata.learning_rate >= 1.0
        """

    @abstractmethod
    def _prepare_model(self)-> None:
        """3. method called during building:
        set the model ready to train (in keras this step is compiling)
        using "specific_var"

        Raises:
            ModelNotDefinedError: if "self.model" is not a Model type
        """

    @abstractmethod
    def train(self, dataset:AiDatasetHandler, metadata:MetaData)-> None:
        """Train the model with "train" and "val"-part of the dataset, with the
        metadata. Before training the model is tested if it exist and ready

        Args:
            dataset (Datasets):
            metadata (MetaData):

        Raises:
            ModelNotDefinedError: if "self.model" is not a Model type
            ModelNotPreparedError: if "self.model" is not compiled or similar
        """

    @abstractmethod
    def predict(
        self,
        X_pred:np.ndarray,
        metadata:MetaData,
        **kwargs)->np.ndarray:
        """return prediction for features X_pred with the metadata
        Before prediction the model is tested if it exist and ready

        Args:
            X_pred (np.ndarray): array-like of shape (n_samples, :)
                                input values
            metadata (MetaData)

        Raises:
            ModelNotDefinedError: if "self.model" is not a Model type
            ModelNotPreparedError: if "self.model" is not compiled or similar

        Returns:
            np.ndarray: array-like of shape (n_samples, :)
                        predicted samples values
        """

    @abstractmethod
    def save(self, metadata:MetaData)-> str:
        """Save the current model object

        Args:
            metadata (MetaData)

        Returns:
            str: the saving path which is automatically set using
            metadata.outputdir
        """

    @abstractmethod
    def load(self, metadata:MetaData)-> None:
        """Load a model from the directory metadata.model_saving_path

        Args:
            metadata (MetaData)
        """

# class ListModels(Enum):
#     """"""

if __name__ == "__main__":
    from glob_utils.log.log  import change_level_logging, main_log
    import logging
    main_log()
    change_level_logging(logging.DEBUG)


text:
import numpy as np
import matplotlib.pyplot as plt
import logging
import tqdm
from termcolor import colored

# logger = logging

class Classifier:
    def __init__(self):

        """
        Assume data set already has 1's at it's first column
        """

        self.nfeatures = None
        self.X = None
        self.y = None

    def fit(self, X, y, alpha=0.1, iters=100, regul=100):
        """
        Fit the optimizer to the given dataset
        """

        # Set up trainig environment
        self.nfeatures = X.shape[1]
        assert (X.shape == (y.size, self.nfeatures))
        theta = np.zeros((self.nfeatures,), dtype='float')
        self.X = X
        self.y = y

        for i in tqdm.tqdm(range(iters)):
            cost = self._cost(theta, regul=regul)
            grad = self._grad(theta, regul=regul)
            theta -= grad*alpha

        print(f" Iter {colored(iters, 'yellow')}\t cost={colored(cost, 'green')}")
        self.cost = cost

        self.theta = theta

    def predict(self, X):
        hypo = self.sigmoid(np.matmul(X, self.theta))
        labels = np.zeros(hypo.shape)
        labels[hypo>=0.5] = 1

        return labels

    def probability(self, X):
        return self.sigmoid(np.matmul(X, self.theta))

    def score(self, X, y):
        labels = self.predict(X)
        correct = np.count_nonzero(labels == y)

        size = len(y)
        acc = correct / float(size)
        # print(f"\nscore {correct}/{size}: {acc:0.5f}")
        return acc

    @staticmethod
    def sigmoid(z):
        return 1 /(1+np.exp(-z))

    def plot_decision_boundary(self, ax0, ax1, axis=None):

        if axis is None:
            ax = plt.gca()

        x1 = self.X[:, ax0]
        x2 = self.X[:, ax1]

        # boundary
        xm0, xM0 = min(self.X[:, ax0]) - 1, max(self.X[:, ax0]) + 1
        xm1, xM1 = min(self.X[:, ax1])-1, max(self.X[:, ax1])+1

        YY, XX = np.mgrid[xm1:xM1:100j, xm0:xM0:100j]
        labels = self.predict(
                np.c_[np.ones(XX.size),XX.ravel(), YY.ravel()]
        )

        labels = labels.reshape(XX.shape)
        # print(labels)
        out = ax.contourf(XX, YY, labels,
                cmap=plt.cm.coolwarm, alpha=0.5
        )
        ax.scatter(x1, x2, c=self.y, cmap='coolwarm', edgecolor='k')

    def _cost(self, theta, regul=0):
        """
        X is of shape (m, n)
            m : size of the dataset
            n : number of features (including 1's)

        y is fo shape (n, ) -> y is only the labels

        """
        X = self.X
        y = self.y
        # theta = theta.reshape((self.nfeatures, 1))

        H = ( 1 / (1+np.exp(-np.matmul(X, theta)) ))# [:, 0] # Our hypothesis
        # print("H = ", H, "shape=", H.shape)
        assert H.shape == y.shape
        m = len(y)

        cost = sum (
            -y*np.log(H)-(1-y)*np.log(1-H)
        )
        cost = cost / m  # finally divide the entire thing with number of dataset
        return cost

    def _grad(self, theta, regul=0):
        """
        returns the gradient of the cost function
        """

        X = self.X
        y = self.y
        # theta = theta.reshape((self.nfeatures, 1))

        H = ( 1 / (1+np.exp(-np.matmul(X, theta)) )) # [:, 0] # Our hypothesis
        m = len(y)  # Length of our training set
        n = X.shape[1] # Number of training features

        dtheta = np.zeros((n,), dtype='float')

        for i in range(m):
            for j in range(n):
                dtheta[j] += (H[i]-y[i])*X[i, j]

        dtheta /= m
        return dtheta

    def trim(self):
        self.X = None
        self.y = None

class MulticlassClassifier(Classifier):
    def fit(self, X, y, **kwargs):
        self.nclasses = len(np.unique(y))

        print("\nStarted trainig")
        print(f"Number of unique classes detected {colored(self.nclasses, 'blue')}")
        print("=="*20)

        self.classifiers = {}

        for i in range(self.nclasses):
            ith_label = np.copy(y)
            ith_label[y==i] = 1
            ith_label[y!=i] = 0

            clf = Classifier()

            print(f" Training classifier {colored(i, 'blue')}")
            print("-"*40)
            clf.fit(X, ith_label, **kwargs)
            clf.trim()
            self.classifiers[i] = clf
            print(f" COST ={colored(clf.cost, 'blue')}\n")

    def predict(self, X):
        probabs = np.zeros((X.shape[0], self.nclasses))
        for i in range(self.nclasses):
            probabs[:, i] = self.classifiers[i].probability(X)

        return np.argmax(probabs, axis=1)

if __name__ == '__main__':
    from sklearn.datasets import load_iris
    from sklearn.linear_model import LinearRegression
    from sklearn.model_selection import train_test_split

    # Load the iris dataset from scipy

    DX, Dy = load_iris(return_X_y=True)
    print(DX.shape)

    poly_terms = (
        np.ones((DX.shape[0],)),
        DX,
        DX[:, 0]*DX[:, 0],
        DX[:, 0]*DX[:, 0],
        DX[:, 0]*DX[:, 0],
    )
    DX = np.concatenate(
        (np.ones((DX.shape[0], 1)) , DX),
        axis=1
    )
    train_X, test_X, train_y, test_y = train_test_split(
            DX, Dy, test_size=0.3
    )

    clf = MulticlassClassifier()
    clf.fit(train_X, train_y, iters=500)
    preds = clf.score(test_X, test_y)
    self_score = clf.score(train_X, train_y)
    print("Score is :", preds)
    print("Score on training data:", self_score)

reason: import numpy,from scipy


text:
import numpy as np
import pytest

from birdman import stats

class TestHotelling:
    def test_hotelling(self):
        p, n = 50, 100  # 50 features, 100 draws
        Xrand = np.random.normal(loc=0, scale=0.1, size=(p, n))
        Xreal = np.random.normal(loc=0, scale=0.1, size=(p, n))
        Xreal[0, :] += 10  # increment first feature uniformly by 10

        _, pval = stats._hotelling(Xrand)
        assert pval > 0.01
        _, pval = stats._hotelling(Xreal)
        assert pval < 0.05

    def test_hotelling_feat_gt_samp(self):
        p, n = 102, 100  # 102 features, 100 draws
        Xrand = np.random.randn(p, n)

        with pytest.raises(ValueError) as excinfo:
            stats._hotelling(Xrand)

        exp_msg = "Number of samples must be larger than number of features!"
        assert str(excinfo.value) == exp_msg

max_stars_repo_name: mortonjt/BIRDMAn

max_issues_repo_name: mortonjt/BIRDMAn

max_forks_repo_name: mortonjt/BIRDMAn


text:
from hydroDL.data import dbBasin
import json
import os
from hydroDL import kPath
import numpy as np

sd = '1982-01-01'
ed = '2018-12-31'
dataName = 'G200'
dictSiteName = 'dict{}.json'.format(dataName)
dirSel = os.path.join(kPath.dirData, 'USGS', 'inventory', 'siteSel')
with open(os.path.join(dirSel, dictSiteName)) as f:
    dictSite = json.load(f)
siteNoLst = dictSite['rmTK']
# DF = dbBasin.DataFrameBasin.new(dataName, siteNoLst, sdStr=sd, edStr=ed)
DF = dbBasin.DataFrameBasin(dataName)

# normalization
DFN = dbBasin.func.localNorm(DF, subset='all')
DFN.saveAs(dataName+'N')

max_stars_repo_name: fkwai/geolearn

max_issues_repo_name: fkwai/geolearn

max_forks_repo_name: fkwai/geolearn


text:
from ..imports import *
def subfinder(mylist, pattern):
    matches = []
    for i in range(len(mylist)):
        if mylist[i] == pattern[0] and mylist[i:i+len(pattern)] == pattern:
            matches.append(pattern)
    return matches

DTOK_TREEBANK=None
def detokenize_treebank(x):
	global DTOK_TREEBANK
	if DTOK_TREEBANK is None:
		from nltk.tokenize.treebank import TreebankWordDetokenizer
		DTOK_TREEBANK = TreebankWordDetokenizer()
	return DTOK_TREEBANK.detokenize(x)

def detokenize(x):
	x=detokenize_treebank(x)
	return x

def setindex(df,key=LINEKEY,badcols={'index','level_0'},sort=True):
	df = df[set(df.columns) - set(df.index.names)]
	df = resetindex(df)

	cols=[]
	for x in key:
		if not x in set(cols) and x in set(df.columns):
			cols.append(x)

	odf=df.set_index(cols)
	if sort: odf=odf.sort_index()
	odf=odf[[col for col in odf.columns if not col in badcols and col not in key]]

	constraints = [c for c in odf.columns if c.startswith('*')]
	resortcols = ['*total'] + [c for c in constraints if c!='*total']
	resortcols+= list(sorted([c for c in odf.columns if c not in set(constraints)]))
	return odf[[c for c in resortcols if c in odf.columns]]

def resetindex(df,badcols={'level_0','index'},**y):
	cols=set(df.columns)
	inds=set(df.index.names)
	both=cols&inds
	# print(cols)
	# print(inds)
	# print(both)
	newdf=df[cols - both - badcols].reset_index(**y)
	# print(newdf.columns)
	# print()
	return newdf

def occurrences(string, sub):
	count = start = 0
	while True:
		start = string.find(sub, start) + 1
		if start > 0:
			count+=1
		else:
			return count

import numpy as np
def rolling_slices(df,window_len=3,incl_empty=True,keep_last_keys=[]):
	nrad=(window_len - 1)//2
	for i in range(len(df)):
		mini = i-nrad
		maxi = i+nrad
		empty_row=pd.Series(dict((k,np.nan) for k in df.columns))
		for xx in keep_last_keys: empty_row[xx]=df.iloc[0][xx]
		irows=[]
		inames=[]
		for ii in range(mini,maxi+1):
			inames+=[ii]
			if ii<0 or ii>=len(df):
				if incl_empty:

					irows+=[empty_row]
			else:
				irows+=[df.iloc[ii]]
		yield pd.DataFrame(irows,index=inames)
def chunks(lst, n):
	"""Yield successive n-sized chunks from lst."""
	for i in range(0, len(lst), n):
		yield lst[i:i + n]

# def product(*args):
# 	if not args:
# 		return iter(((),)) # yield tuple()
# 	return (items + (item,)
# 		for items in product(*args[:-1]) for item in args[-1])

def hashstr(*x):
	import hashlib
	return hashlib.sha224(str(x).encode('utf-8')).hexdigest()

def pmap_do(inp):
	func,obj,args,kwargs = inp
	return func(obj,*args,**kwargs)

def pmap_iter(func, objs, args=[], kwargs={}, num_proc=DEFAULT_NUM_PROC, use_threads=False, progress=True, desc=None, **y):
	"""
	Yields results of func(obj) for each obj in objs
	Uses multiprocessing.Pool(num_proc) for parallelism.
	If use_threads, use ThreadPool instead of Pool.
	Results in any order.
	"""

	# imports
	import multiprocessing as mp
	from tqdm import tqdm

	# check num proc
	num_cpu = mp.cpu_count()
	if num_proc>num_cpu: num_proc=num_cpu

	# if parallel
	if not desc: desc=f'Mapping {func.__name__}()'
	if desc: desc=f'{desc} [x{num_proc}]'
	if num_proc>1 and len(objs)>1:

		# real objects
		objects = [(func,obj,args,kwargs) for obj in objs]

		# create pool
		pool=mp.Pool(num_proc) if not use_threads else mp.pool.ThreadPool(num_proc)

		# yield iter
		iterr = pool.imap(pmap_do, objects)

		for res in tqdm(iterr,total=len(objects),desc=desc) if progress else iterr:
			yield res

		# Close the pool?
		pool.close()
		pool.join()
	else:
		# yield
		for obj in (tqdm(objs,desc=desc) if progress else objs):
			yield func(obj,*args,**kwargs)

def pmap(*x,**y):
	"""
	Non iterator version of pmap_iter
	"""
	# return as list
	return list(pmap_iter(*x,**y))

def index_by_truth(x):
    i=0
    for y in x:
        if y:
            yield i
            i+=1
        else:
            yield np.nan

def do_pmap_group(obj,*args,**kwargs):
	import pandas as pd
	import types
	func,group_df,group_key,group_name = obj

	if type(group_name) not in {list,tuple}:group_name=[group_name]
	if type(group_df)==str: group_df=pd.read_pickle(group_df)
	out=func(group_df,*args,**kwargs)
	if isinstance(out, types.GeneratorType):
		out=[x for x in out]
	if type(out)==list:
		try:
			out=[x for x in out if type(x) in {pd.DataFrame, pd.Series}]
			out=pd.concat(out)
		except ValueError:
			return out
	if type(out)==pd.DataFrame:
		for x,y in zip(group_key,group_name): out[x]=y
	return out

def joindfs(a,b):
    bcols = set(b.columns) - set(a.columns)
    return a.join(b[bcols])

def slices(l,n,strict=True):
    o=[]
    for x in l:
        o.append(x)
        if len(o)>n: o.pop(0)
        if not strict or len(o)==n:
            yield list(o)
def apply_combos(df,group1,group2,combo_key='combo_i'):
    # combo of indices?
    combo_opts = [
        [x for ii,x in grp.groupby(group2)]
        for i,grp in df.groupby(group1)
    ]

    # poss
    for combo in product(*combo_opts):
        yield pd.concat(combo)
def pmap_iter_groups(func,df_grouped,use_cache=False,num_proc=DEFAULT_NUM_PROC,iter=False,**attrs):
	import os,tempfile,pandas as pd
	from tqdm import tqdm

	# get index/groupby col name(s)
	group_key=df_grouped.grouper.names
	# if not using cache
	# if not use_cache or attrs.get('num_proc',1)<2:
	if not use_cache or len(df_grouped)<2 or num_proc<2:
		objs=[
			(func,group_df,group_key,group_name)
			for group_name,group_df in df_grouped
		]
	else:
		objs=[]
		tmpdir=tempfile.mkdtemp()
		# for i,(group_name,group_df) in enumerate(tqdm(list(df_grouped),desc='Preparing input')):
		for i,(group_name,group_df) in enumerate(df_grouped):
			tmp_path = os.path.join(tmpdir, str(i)+'.pkl')
			# print([i,group_name,tmp_path,group_df])
			group_df.to_pickle(tmp_path)
			objs+=[(func,tmp_path,group_key,group_name)]

	# desc?
	if not attrs.get('desc'): attrs['desc']=f'Mapping {func.__name__}'

	#iterfunc = pmap if not iter else pmap_iter
	#return pd.concat(iterfunc) if not iter else
	return pmap_iter(
		do_pmap_group,
		objs,
		num_proc=num_proc,
		**attrs
	)
def pmap_groups(*x,**y):
	res = list(pmap_iter_groups(*x,**y))
	# print([type(x) for x in res])
	# for y in res[-1]:
		# print(type(y), y)
	resl = []
	for x in res:
		if type(x)==list:
			for y in x:
				resl.append(y)
		else:
			resl.append(x)
	return pd.concat(resl)

def check_basic_config():
	# check basic config
	if not os.path.exists(PATH_HOME): os.makedirs(PATH_HOME)
	if not os.path.exists(PATH_DATA): os.makedirs(PATH_DATA)
	if not os.path.exists(os.path.join(PATH_DATA,'en')):
		zipfn=os.path.join(PATH_HOME,'data_cadence.zip')
		download(DATA_URL, zipfn)
		unzip(zipfn, PATH_HOME)
	try:
		nltk.word_tokenize('testing')
	except LookupError:
		nltk.download('punkt')
	try:
		nltk.corpus.stopwords.words('english')
	except LookupError:
		nltk.download('stopwords')

### utils
def printm(x):
	from IPython.display import display,Markdown
	display(Markdown(x))

def download_wget(url, save_to, **attrs):
	import wget
	save_to_dir,save_to_fn=os.path.split(save_to)
	if save_to_dir:
		if not os.path.exists(save_to_dir): os.makedirs(save_to_dir)
		os.chdir(save_to_dir)
	fn=wget.download(url,bar=wget.bar_adaptive)
	os.rename(fn,save_to_fn)
	# print('\n>> saved:',save_to)

def download(url,save_to,force=False,desc=''):
	here=os.getcwd()
	download_wget(url,save_to,desc=desc)
	os.chdir(here)

def unzip(zipfn, dest='.', flatten=False, overwrite=False, replace_in_filenames={},desc='',progress=True):
	from zipfile import ZipFile
	from tqdm import tqdm

	# Open your .zip file
	if not desc: desc=f'Extracting {os.path.basename(zipfn)}'
	with ZipFile(zipfn) as zip_file:
		namelist=zip_file.namelist()

		# Loop over each file
		iterr=tqdm(iterable=namelist, total=len(namelist),desc=desc) if progress else namelist
		for member in iterr:
			# Extract each file to another directory
			# If you want to extract to current working directory, don't specify path
			filename = os.path.basename(member)
			if not filename: continue
			target_fnfn = os.path.join(dest,member) if not flatten else os.path.join(dest,filename)
			for k,v in replace_in_filenames.items(): target_fnfn = target_fnfn.replace(k,v)
			if not overwrite and os.path.exists(target_fnfn): continue
			target_dir = os.path.dirname(target_fnfn)
			try:
				if not os.path.exists(target_dir): os.makedirs(target_dir)
			except FileExistsError:
				pass
			except FileNotFoundError:
				continue

			with zip_file.open(member) as source, open(target_fnfn,'wb') as target:
				shutil.copyfileobj(source, target)

def get_num_lines(filename):
	from smart_open import open

	def blocks(files, size=65536):
		while True:
			b = files.read(size)
			if not b: break
			yield b

	with open(filename, 'r', errors='ignore') as f:
		numlines=sum(bl.count("\n") for bl in blocks(f))

	return numlines

max_stars_repo_name: tomaarsen/cadence

max_issues_repo_name: tomaarsen/cadence

max_forks_repo_name: tomaarsen/cadence


text:
import numpy as np

def template_popularity(model, test_fps, train_labels, test_appl_labels, topk=100, batch_size=512):
    class_counts = np.bincount(train_labels)
    pred = model.predict(test_fps)
    ind = np.argsort(-pred)[:, :topk]
    appl_ind = [i[np.isin(i, np.argwhere(appl_label.A))] for i, appl_label in zip(ind, test_appl_labels)]
    appl_pop = np.array([class_counts[i] for i in appl_ind])
    max_length = max([len(a) for a in appl_pop])
    appl_pop = np.vstack([np.pad(pop.astype(float), pad_width=(0, max_length-len(pop)), mode='constant', constant_values=np.NaN) for pop in appl_pop])
    return appl_pop

max_stars_repo_name: ml-jku/mhn-react

max_issues_repo_name: ml-jku/mhn-react

max_forks_repo_name: ml-jku/mhn-react


text:
from .abc_interpreter import Interpreter, InputGradientInterpreter
from ..data_processor.readers import preprocess_inputs, preprocess_save_path
from ..data_processor.visualizer import explanation_to_vis, show_vis_explanation, save_image

import numpy as np
import paddle

class GradShapCVInterpreter(InputGradientInterpreter):
    """
    Gradient SHAP Interpreter for CV tasks.

    More details regarding the GradShap method can be found in the original paper:
    http://papers.nips.cc/paper/7062-a-unified-approach-to-interpreting-model-predictions
    """

    def __init__(self,
                 paddle_model,
                 use_cuda=None,
                 device='gpu:0',
                 model_input_shape=[3, 224, 224]) -> None:
        """

        Args:
            paddle_model (callable): A model with ``forward`` and possibly ``backward`` functions.
            device (str): The device used for running `paddle_model`, options: ``cpu``, ``gpu:0``, ``gpu:1`` etc.
            use_cuda (bool):  Would be deprecated soon. Use ``device`` directly.
            model_input_shape (list, optional): The input shape of the model. Default: [3, 224, 224]
        """
        InputGradientInterpreter.__init__(self, paddle_model, device, use_cuda)
        self.model_input_shape = model_input_shape

    def interpret(self,
                  inputs,
                  labels=None,
                  baselines=None,
                  n_samples=5,
                  noise_amount=0.1,
                  visual=True,
                  save_path=None):
        """
        Main function of the interpreter.

        Args:
            inputs (str or list of strs or numpy.ndarray): The input image filepath or a list of filepaths or numpy array of read images.
            labels (list or tuple or numpy.ndarray, optional): The target labels to analyze. The number of labels should be equal to the number of images. If None, the most likely label for each image will be used. Default: None            baseline (numpy.ndarray, optional): The baseline input. If None, all zeros will be used. Default: None
            baselines (numpy.ndarray or None, optional): The baseline images to compare with. It should have the same shape as images and same length as the number of images.
                                                        If None, the baselines of all zeros will be used. Default: None.
            n_samples (int, optional): The number of randomly generated samples. Default: 5.
            noise_amount (float, optional): Noise level of added noise to each image.
                                            The std of Guassian random noise is noise_amount * (x_max - x_min). Default: 0.1
            visual (bool, optional): Whether or not to visualize the processed image. Default: True.
            save_path (str or list of strs or None, optional): The filepath(s) to save the processed image(s). If None, the image will not be saved. Default: None

        Returns:
            [numpy.ndarray]: interpretations for images
        """

        imgs, data = preprocess_inputs(inputs, self.model_input_shape)
        bsz = len(data)
        self.data_type = np.array(data).dtype

        self._build_predict_fn(gradient_of='probability')

        if labels is None:
            _, labels = self.predict_fn(data, labels)

        def add_noise_to_inputs(data):
            max_axis = tuple(np.arange(1, data.ndim))
            stds = noise_amount * (
                np.max(data, axis=max_axis) - np.min(data, axis=max_axis))
            noise = np.concatenate([
                np.random.normal(0.0, stds[j], (n_samples, ) + tuple(d.shape))
                for j, d in enumerate(data)
            ]).astype(self.data_type)
            repeated_data = np.repeat(data, (n_samples, ) * len(data), axis=0)
            return repeated_data + noise

        data_with_noise = add_noise_to_inputs(data)

        if baselines is None:
            baselines = np.zeros_like(data)
        baselines = np.repeat(baselines, (n_samples, ) * bsz, axis=0)

        labels = np.array(labels).reshape(
            (bsz, 1))  #.repeat(n_samples, axis=0)
        labels = np.repeat(labels, (n_samples, ) * bsz, axis=0)

        rand_scales = np.random.uniform(
            0.0, 1.0, (bsz * n_samples, 1)).astype(self.data_type)

        input_baseline_points = np.array([
            d * r + b * (1 - r)
            for d, r, b in zip(data_with_noise, rand_scales, baselines)
        ])

        gradients, _ = self.predict_fn(input_baseline_points, labels)

        input_baseline_diff = data_with_noise - baselines
        explanations = gradients * input_baseline_diff

        explanations = np.concatenate([
            np.mean(
                explanations[i * n_samples:(i + 1) * n_samples],
                axis=0,
                keepdims=True) for i in range(bsz)
        ])

        # visualization and save image.
        save_path = preprocess_save_path(save_path, bsz)
        for i in range(bsz):
            vis_explanation = explanation_to_vis(imgs[i], np.abs(explanations[i]).sum(0), style='overlay_grayscale')
            if visual:
                show_vis_explanation(vis_explanation)
            if save_path[i] is not None:
                save_image(save_path[i], vis_explanation)

        return explanations

class GradShapNLPInterpreter(Interpreter):
    """
    Gradient SHAP Interpreter for NLP tasks.

    More details regarding the GradShap method can be found in the original paper:
    http://papers.nips.cc/paper/7062-a-unified-approach-to-interpreting-model-predictions
    """

    def __init__(self, paddle_model, use_cuda=True, device='gpu:0') -> None:
        """

        Args:
            paddle_model (callable): A paddle model that outputs predictions.
            use_cuda (bool, optional): Whether or not to use cuda. Default: True
            model_input_shape (list, optional): The input shape of the model. Default: [3, 224, 224]
        """
        Interpreter.__init__(self, paddle_model, device, use_cuda)

        self.use_cuda = use_cuda
        self.paddle_prepared = False

    def interpret(self,
                  data,
                  labels=None,
                  n_samples=5,
                  noise_amount=0.1,
                  return_pred=False,
                  visual=True):
        """
        Main function of the interpreter.

        Args:
            data (tuple or paddle.tensor): The inputs to the NLP model.
            label (list or numpy.ndarray, optional): The target label to analyze. If None, the most likely label will be used. Default: None.
            n_samples (int, optional): The number of randomly generated samples. Default: 5.
            noise_amount (float, optional): Noise level of added noise to the embeddings.
                                            The std of Guassian random noise is noise_amount * (x_max - x_min). Default: 0.1
            return_pred (bool, optional): Whether or not to return predicted labels and probabilities. If True, a tuple of predicted labels, probabilities, and interpretations will be returned.
                                        There are useful for visualization. Else, only interpretations will be returned. Default: False.
            visual (bool, optional): Whether or not to visualize. Default: True.

        Returns:
            [numpy.ndarray or tuple]: interpretations for each word or a tuple of predicted labels, probabilities, and interpretations
        """

        self.noise_amount = noise_amount
        if not self.paddle_prepared:
            self._paddle_prepare()

        if isinstance(data, tuple):
            n = data[0].shape[0]
        else:
            n = data.shape[0]

        self._alpha = 1
        gradients, out, embedding = self.predict_fn(data, [0] * n)
        out = paddle.nn.functional.softmax(paddle.to_tensor(out)).numpy()

        if labels is None:
            labels = np.argmax(out, axis=1)
        else:
            labels = np.array(labels)

        labels = labels.reshape((n, ))

        rand_scales = np.random.uniform(0.0, 1.0, (n_samples, ))

        total_gradients = np.zeros_like(gradients)
        for alpha in rand_scales:
            self._alpha = alpha
            gradients, _, _ = self.predict_fn(data, labels)
            total_gradients += np.array(gradients)

        avg_gradients = total_gradients / n_samples
        interpretations = avg_gradients * embedding

        if return_pred:
            pred_labels = labels.reshape((n, ))
            pred_probs = [
                p[pred_labels[i]] for i, p in enumerate(np.array(out))
            ]
            return (pred_labels, pred_probs, interpretations)

        return interpretations

    def _paddle_prepare(self, predict_fn=None):
        if predict_fn is None:
            paddle.set_device(self.device)

            self.paddle_model.train()

            self._embedding = None

            def hook(layer, input, output):
                output += paddle.normal(
                    std=self.noise_amount, shape=output.shape)
                output = self._alpha * output
                self._embedding = output
                return output

            for n, v in self.paddle_model.named_sublayers():
                if "embedding" in v.__class__.__name__.lower():
                    v.register_forward_post_hook(hook)
                if "batchnorm" in v.__class__.__name__.lower():
                    v._use_global_stats = True
                if "dropout" in v.__class__.__name__.lower():
                    v.p = 0

            def predict_fn(data, labels):
                if isinstance(data, tuple):
                    probs = self.paddle_model(*data)
                else:
                    probs = self.paddle_model(data)
                labels_onehot = paddle.nn.functional.one_hot(
                    paddle.to_tensor(labels), num_classes=probs.shape[1])
                target = paddle.sum(probs * labels_onehot, axis=1)
                target.backward()
                gradients = self._embedding.grad
                if isinstance(gradients, paddle.Tensor):
                    gradients = gradients.numpy()
                return gradients, probs.numpy(), self._embedding.numpy(
                )

        self.predict_fn = predict_fn
        self.paddle_prepared = True

max_stars_repo_name: PaddlePaddle/InterpretDL

max_issues_repo_name: XianrenYty/InterpretDL

max_forks_repo_name: XianrenYty/InterpretDL


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from posixpath import split
import rospy
from geometry_msgs.msg import Twist
from geometry_msgs.msg import PoseWithCovarianceStamped
from sensor_msgs.msg import LaserScan
import tf
import actionlib
from move_base_msgs.msg import MoveBaseAction, MoveBaseGoal
import actionlib_msgs

import math
import json

# Ref: https://hotblackrobotics.github.io/en/blog/2018/01/29/action-client-py/

from std_msgs.msg import Bool, String, Int32, Float64, Int32MultiArray
import numpy as np

from visualization_msgs.msg import Marker

TargetKey = [   "Tomato_N",
                "Tomato_S",
                "Omelette_N",
                "Omelette_S",
                "Pudding_N",
                "Pudding_S",
                "OctopusWiener_N",
                "OctopusWiener_S",
                "FriedShrimp_N",
                "FriedShrimp_E",
                "FriedShrimp_W",
                "FriedShrimp_S"
            ]

class WarStatus():

    def __init__(self):

        # publisher
        self.score_status = rospy.Publisher('score_status', Int32MultiArray, queue_size=10)

        # subscriber
        self.state_sub = rospy.Subscriber('war_state', String, self.StateCallback)

        self.json_score = ""
        self.score_array = []

    def StateCallback(self, data):
        self.score_array = []
        self.json_score = json.loads(data.data)

        for i in range(0, len(self.json_score["targets"])):
            for j , key in enumerate(TargetKey):
                if(self.json_score["targets"][i]["name"] == key):
                    if self.json_score["targets"][i]["player"] == rospy.get_param("/side"):
                        self.score_array.append(1)
                    else:
                        self.score_array.append(0)

        # print(self.score_array)
        # rospy.loginfo("SCORE: {}".format(self.score_array))
        self.score_status.publish(Int32MultiArray(data = self.score_array))

if __name__ == '__main__':
    try:
        rospy.init_node('score_status', anonymous=False)
        status = WarStatus()

        rate = rospy.Rate(10) # 10hz

        while not rospy.is_shutdown():
            rate.sleep()

    except rospy.ROSInterruptException:
        pass


text:
"""
  genetic_salesman.py
  Heuristic travelling salesman solver using genetic optimisation
"""

import random as rng
import math
import numpy as np

import matplotlib.pyplot as plt
from matplotlib.animation import FuncAnimation

np.random.seed(1)

class NodeList(object):
  """
    A set of nodes implemented with complex numbers.
    Plotted in a rough circle
    The first element is always (0,0)
  """
  def __init__(self, count, noise_rate = .45):
    """Initialisation. Sets the xy coords of the nodes in a vector self.P
    """
    self.count = count

    angle = np.arange(count)/count * 2*np.pi * 1j
    noise = noise_rate*(1-2*np.random.random(count) + 1j*(1-2*np.random.random(count)))

    self.P = np.exp(angle) + noise

  def path_length(self, paths):
    """ The total distance for given paths (as rows) indicated as a list of indices.
      Wraps around to first point
    """
    # Wraparound:
    paths = np.hstack([paths[:,:], paths[:,0:1]])

    P_order = self.P[paths]
    dists = np.absolute(P_order[:,1:]-P_order[:,:-1])

    return np.sum(dists, axis=1, keepdims = True)

  def plot(self, ax, path = None):
    """Plotting the nodes and optionally a path on a 2d grid
    """
    path = np.hstack([path, path[0:1]])

    x, y = self.P.real, self.P.imag
    ax.plot(x,y, 'o')
    ax.plot(x[path], y[path], '-')

class GeneticOptimiser(object):
  """Genetic optimiser for TSP
    self.population contains gentypes (one per row)
  """
  def __init__(self, gentype_lngth, popsize = 1000, cost_fun = None):
    self.popsize = popsize
    self.gentype_lngth = gentype_lngth
    self.cost_fun = cost_fun

    self.costs = np.empty(self.popsize).reshape((self.popsize, 1))
    self.age = 0

    self.params = {
        'perc_top'        : .003,
        'max_crossover'   : .5,
        'tournament_size' : 10,

    }

    self.initialise_population()
    self.cost_me()

  def initialise_population(self):
    """Initialise the whole population
    every tour starts at 0
    """
    #Permute points 1 to k
    self.population = np.array([np.random.permutation(list(range(1, self.gentype_lngth)))
                                for i in range(self.popsize)])
    # Add zero to front
    self.population = np.hstack([np.zeros((self.popsize, 1), dtype = int), self.population])

  def evolve(self):
    """Perform one optimisation step on the whole population
    Mutation and crossover
    """
    top_index   = list(range(int(self.popsize * self.params['perc_top'])))
    # lucky_index = [s for s in np.random.permutation(range(int(self.popsize)))[:len(top_index)]]
    lucky_index = list(np.random.choice(range(self.popsize), size=len(top_index), replace = False))

    mutagen = np.random.choice(['randswap', 'pickinsert', 'neighswap', 'partreverse'])
    newpop = np.vstack([
        # Mutate
        self.mutate(top_index, mutagen),
        self.mutate(lucky_index, 'pickinsert'), #pickinsert is the better choice for poor agents
        # Crossover
        self.crossover(top_index, np.random.permutation(top_index)),
        self.crossover(lucky_index, top_index),

      ])

    # Add new members
    oldsize = self.popsize
    self.add_to_population(newpop)
    self.trim_population(oldsize)

    self.age += 1

  def add_to_population(self, newpop):
    """Introduce new agents to an existing population
    """

    # Add to population and cost array
    self.population = np.vstack([self.population, newpop])
    self.popsize += len(newpop)
    newcosts = np.empty(len(newpop)).reshape((len(newpop), 1))
    self.costs = np.vstack([self.costs, newcosts])

    # Recalc costs
    self.cost_me(list(range(self.popsize-len(newpop), self.popsize)))

  def trim_population(self, newsize, tournament = False):
    """ Remove the poorest agents from a population
    TODO: Add tournament selection
    """
    if tournament:
      num_of_tours = self.popsize // self.params['tournament_size']
      to_keep_per_tour = newsize // num_of_tours

    if not tournament: # Simple trim from the end
      self.population = self.population[:newsize, :]
      self.costs = self.costs[:newsize, :]

    self.popsize = newsize

  def cost_me(self, indices = None):
    """Calculate costs for the gentypes in indices
    Reorders the whole population to reflect the new ranking
    """
    if not indices: indices = list(range(self.popsize))
    # Calculate costs
    self.costs[indices] = self.cost_fun(self.population[indices]) #mind column vector

    # Reorder arrays
    sort_index = np.argsort(self.costs.flatten())  # Reverse order
    self.costs = self.costs[sort_index, :]
    self.population = self.population[sort_index,:]

  def mutate(self,indices, mutagen = 'randswap'):
    """
    Performs one of three mutations on a subset
      >randswap: Pick two elmnts at random and trade places
      >pickinsrt: Grap one rndm elemnt and relocate it to a rndm location
      >neighswap: Pick two adjacent elmnts and swap

    Mutations do not ever leave the agent unchanged (to avoid a single agent dominating)
    """
    if not indices: indices = list(range(self.popsize))

    assert mutagen in ['randswap', 'pickinsert', 'neighswap', 'partreverse']
    newpop = self.population[indices, :].copy()

    # swap random columns
    if mutagen == 'randswap':
      a,b = np.random.choice(range(self.gentype_lngth), size=(2,1), replace = False).flatten() # Pick two seperate
      newpop[:,a], newpop[:,b] = self.population[indices,b], self.population[indices,a]

    # pick 'n' insert
    if mutagen == 'pickinsert':
      pck,ins = np.random.choice(range(self.gentype_lngth), size=(2,1), replace = False).flatten() # Pick two seperate
      newindex = list(range(self.gentype_lngth))
      newindex.insert(ins, newindex.pop(pck)) # pop and place
      newpop = self.population[np.ix_(indices,newindex)]

    # swap neighbours
    if mutagen == 'neighswap':
      a = np.random.randint(1, high=self.gentype_lngth, size=1, dtype=int)[0]
      newpop[:,a], newpop[:,a-1] = self.population[indices,a-1], self.population[indices,a]

    #reverse a part of the path
    if mutagen == 'partreverse':
      max_crossover = self.params['max_crossover']
      genlngth = self.gentype_lngth

      start = np.random.randint(0,genlngth-1)
      end = min(genlngth, start + np.random.randint(1,int(genlngth*max_crossover)))
      new_index = list(range(start)) + list(range(start, end))[::-1] + list(range(end, genlngth))

      newpop = self.population[np.ix_(indices,new_index)]

    # roll zeros back to front
    zero_ind =  np.argwhere(newpop[0,:] == 0)[0,0]
    new_index = list(range(zero_ind,self.gentype_lngth)) + list(range(0,zero_ind))
    newpop = newpop[:,new_index]

    return newpop

  def crossover(self, left_inds, right_inds):
    """Crossover a pair of agents A and B
      Move a part of the genotype from B to A
      Remove doublings
    """

    # Define index range to cross from right to left
    max_crossover = self.params['max_crossover']
    genlngth = self.gentype_lngth

    start = np.random.randint(1,genlngth-1)
    end = min(genlngth, start + np.random.randint(1,int(genlngth*max_crossover)))
    cross_index = list(range(start, end))

    # Grab part from right, make negative as label
    cross_part = -self.population[np.ix_(right_inds, cross_index)]

    # Insert into left
    full_list = np.hstack([self.population[left_inds, :start], cross_part, self.population[left_inds, start:]])
    newpop = np.empty(self.population[left_inds,:].shape, dtype = int)
    # Remove positive matches
    for i, ind in enumerate(left_inds):
      # Keep all items not in cross_part
      newpop[i,:]= full_list[i, np.logical_not(np.isin(full_list[i,:], -cross_part[i, :]))]
    newpop = np.absolute(newpop)

    return newpop

  def plot_cost_profile(self, ax):
    ax.hist(self.costs, bins = 50)
    # ax.set_yscale('log')
    # ax.set_xscale('log')

#
# Profiler set up
#

def profile_setup():
  n_prof = 500000
  k_prof = 32
  N_prof = NodeList(k_prof, .45)
  G_prof = GeneticOptimiser(gentype_lngth = k_prof, popsize = n_prof, cost_fun = N_prof.path_length)
  return G_prof

def profile_me(G_prof):
  for i in range(50):
    G_prof.evolve()

#
# MAIN set-up
#

if __name__ == '__main__':

  k = 32  # Node count
  n = 5000   # Population size
  evo_steps = 50
  num_of_reports = 10

  N = NodeList(k, .45)

  G = GeneticOptimiser(gentype_lngth = k, popsize = n, cost_fun = N.path_length)

  for stp in range(evo_steps+1):

    if stp % (evo_steps // num_of_reports) == 0:

      print('{}: {}'.format(stp, G.costs[0]))

      f2, [ax1, ax2] = plt.subplots(1,2)
      N.plot(ax2, G.population[0])
      ax2.set_xlim(-1.6, 1.6)
      ax2.set_ylim(-1.6, 1.6)
      ax2.set_aspect(1)
      G.plot_cost_profile(ax1)
      f2.savefig('{:4d}.jpg'.format(stp))
      plt.axes().set_aspect('equal', 'datalim')
      plt.close()

    G.evolve()

max_stars_repo_name: randompirate/genetic-salesman

max_issues_repo_name: randompirate/genetic-salesman

max_forks_repo_name: randompirate/genetic-salesman