text:
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
    This package implement Graph structure for handling graph data.
"""

import os
import json
import copy
import warnings
from collections import defaultdict

import numpy as np
import paddle
import paddle.distributed as dist

from pgl.utils import op
import pgl.graph_kernel as graph_kernel
from pgl.message import Message
from pgl.utils.edge_index import EdgeIndex
from pgl.utils.helper import check_is_tensor, scatter, maybe_num_nodes
from pgl.utils.helper import generate_segment_id_from_index, unique_segment

try:
    from paddle.incubate import graph_send_recv
except:
    from pgl.utils.helper import graph_send_recv

class Graph(object):
    """Implementation of graph interface in pgl.

    This is a simple implementation of graph structure in pgl.

    `pgl.Graph` is an alias for `pgl.graph.Graph`

    Args:

        edges: list of (u, v) tuples, 2D numpy.ndarray or 2D paddle.Tensor.

        num_nodes (optional: int, numpy or paddle.Tensor): Number of nodes in a graph.
                           If not provided, the number of nodes will be infered from edges.

        node_feat (optional): a dict of numpy array as node features.

        edge_feat (optional): a dict of numpy array as edge features (should
                                have consistent order with edges).

    Examples 1:

        - Create a graph with numpy.
        - Convert it into paddle.Tensor.
        - Do send recv for graph neural network.

        .. code-block:: python

            import numpy as np
            import pgl

            num_nodes = 5
            edges = [ (0, 1), (1, 2), (3, 4)]
            feature = np.random.randn(5, 100).astype(np.float32)
            edge_feature = np.random.randn(3, 100).astype(np.float32)
            graph = pgl.Graph(num_nodes=num_nodes,
                        edges=edges,
                        node_feat={
                            "feature": feature
                        },
                        edge_feat={
                            "edge_feature": edge_feature
                        })
            graph.tensor()

            model = pgl.nn.GCNConv(100, 100)
            out = model(graph, graph.node_feat["feature"])

    Examples 2:

        - Create a graph with paddle.Tensor.
        - Do send recv for graph neural network.

        .. code-block:: python

            import paddle
            import pgl

            num_nodes = 5
            edges = paddle.to_tensor([ (0, 1), (1, 2), (3, 4)])
            feature = paddle.randn(shape=[5, 100])
            edge_feature = paddle.randn(shape=[3, 100])
            graph = pgl.Graph(num_nodes=num_nodes,
                        edges=edges,
                        node_feat={
                            "feature": feature
                        },
                        edge_feat={
                            "edge_feature": edge_feature
                        })

            model = pgl.nn.GCNConv(100, 100)
            out = model(graph, graph.node_feat["feature"])

    """

    def __init__(self,
                 edges,
                 num_nodes=None,
                 node_feat=None,
                 edge_feat=None,
                 **kwargs):
        if node_feat is not None:
            self._node_feat = node_feat
        else:
            self._node_feat = {}

        if edge_feat is not None:
            self._edge_feat = edge_feat
        else:
            self._edge_feat = {}

        if not check_is_tensor(edges):
            if isinstance(edges, np.ndarray):
                if edges.dtype != "int64":
                    edges = edges.astype("int64")
            else:
                edges = np.array(edges, dtype="int64")

        self._edges = edges

        if num_nodes is None:
            self._num_nodes = maybe_num_nodes(self._edges)
        else:
            self._num_nodes = num_nodes

        self._adj_src_index = kwargs.get("adj_src_index", None)
        self._adj_dst_index = kwargs.get("adj_dst_index", None)

        if check_is_tensor(self._num_nodes, self._edges,
                           *list(self._node_feat.values()),
                           *list(self._edge_feat.values())):
            self._is_tensor = True
        elif self._adj_src_index is not None and self._adj_src_index.is_tensor(
        ):
            self._is_tensor = True
        elif self._adj_dst_index is not None and self._adj_dst_index.is_tensor(
        ):
            self._is_tensor = True
        else:
            self._is_tensor = False

        if self._is_tensor:
            # ensure all variable is tenosr
            if not check_is_tensor(self._num_nodes):
                self._num_nodes = paddle.to_tensor(self._num_nodes)

            if not check_is_tensor(self._edges):
                self._edges = paddle.to_tensor(self._edges)

            for key in self._node_feat:
                if not check_is_tensor(self._node_feat[key]):
                    self._node_feat[key] = paddle.to_tensor(self._node_feat[
                        key])

            for key in self._edge_feat:
                if not check_is_tensor(self._edge_feat[key]):
                    self._edge_feat[key] = paddle.to_tensor(self._edge_feat[
                        key])

            if self._adj_src_index is not None:
                if not self._adj_src_index.is_tensor():
                    self._adj_src_index.tensor(inplace=True)

            if self._adj_dst_index is not None:
                if not self._adj_dst_index.is_tensor():
                    self._adj_dst_index.tensor(inplace=True)

        # preprocess graph level informations
        self._process_graph_info(**kwargs)
        self._nodes = None

    def recv(self, reduce_func, msg, recv_mode="dst"):
        """Recv message and aggregate the message by reduce_func.

        The UDF reduce_func function should has the following format.

        .. code-block:: python

            def reduce_func(msg):
                '''
                    Args:

                        msg: An instance of Message class.

                    Return:

                        It should return a tensor with shape (batch_size, out_dims).
                '''
                pass

        Args:

            msg: A dictionary of tensor created by send function..

            reduce_func: A callable UDF reduce function.

        Return:

            A tensor with shape (num_nodes, out_dims). The output for nodes with
            no message will be zeros.

        """

        if not self._is_tensor:
            raise ValueError("You must call Graph.tensor()")

        if not isinstance(msg, dict):
            raise TypeError(
                "The input of msg should be a dict, but receives a %s" %
                (type(msg)))

        if not callable(reduce_func):
            raise TypeError("reduce_func should be callable")

        src, dst, eid = self.sorted_edges(sort_by=recv_mode)

        msg = op.RowReader(msg, eid)

        if (recv_mode == "dst") and (not hasattr(self, "_dst_uniq_ind")):
            self._dst_uniq_ind, self._dst_segment_ids = unique_segment(dst)
        if (recv_mode == "src") and (not hasattr(self, "_src_uniq_ind")):
            self._src_uniq_ind, self._src_segment_ids = unique_segment(src)

        if recv_mode == "dst":
            uniq_ind, segment_ids = self._dst_uniq_ind, self._dst_segment_ids
        elif recv_mode == "src":
            uniq_ind, segment_ids = self._src_uniq_ind, self._src_segment_ids

        bucketed_msg = Message(msg, segment_ids)
        output = reduce_func(bucketed_msg)
        output_dim = output.shape[-1]
        init_output = paddle.zeros(
            shape=[self._num_nodes, output_dim], dtype=output.dtype)
        final_output = scatter(init_output, uniq_ind, output)

        return final_output

    def __repr__(self):
        """Pretty Print the Graph
        """
        repr_dict = {"class": self.__class__.__name__}
        if self._is_tensor:
            num_nodes = self.num_nodes.numpy()
        else:
            num_nodes = self.num_nodes
        repr_dict["num_nodes"] = int(num_nodes)
        repr_dict["edges_shape"] = self.edges.shape
        repr_dict["node_feat"] = []
        for key, value in self.node_feat.items():
            repr_dict["node_feat"].append({
                "name": key,
                "shape": list(value.shape),
                "dtype": str(value.dtype)
            })

        repr_dict["edge_feat"] = []
        for key, value in self.edge_feat.items():
            repr_dict["edge_feat"].append({
                "name": key,
                "shape": list(value.shape),
                "dtype": str(value.dtype)
            })

        return json.dumps(repr_dict, ensure_ascii=False)

    @classmethod
    def load(cls, path, mmap_mode="r"):
        """Load Graph from path and return a Graph in numpy.

        Args:

            path: The directory path of the stored Graph.

            mmap_mode: Default :code:`mmap_mode="r"`. If not None, memory-map the graph.

        """

        num_nodes = np.load(
            os.path.join(path, 'num_nodes.npy'), mmap_mode=mmap_mode)
        edges = np.load(os.path.join(path, 'edges.npy'), mmap_mode=mmap_mode)
        num_graph = np.load(
            os.path.join(path, 'num_graph.npy'), mmap_mode=mmap_mode)
        if os.path.exists(os.path.join(path, 'graph_node_index.npy')):
            graph_node_index = np.load(
                os.path.join(path, 'graph_node_index.npy'),
                mmap_mode=mmap_mode)
        else:
            graph_node_index = None

        if os.path.exists(os.path.join(path, 'graph_edge_index.npy')):
            graph_edge_index = np.load(
                os.path.join(path, 'graph_edge_index.npy'),
                mmap_mode=mmap_mode)
        else:
            graph_edge_index = None

        if os.path.isdir(os.path.join(path, 'adj_src')):
            adj_src_index = EdgeIndex.load(
                os.path.join(path, 'adj_src'), mmap_mode=mmap_mode)
        else:
            adj_src_index = None

        if os.path.isdir(os.path.join(path, 'adj_dst')):
            adj_dst_index = EdgeIndex.load(
                os.path.join(path, 'adj_dst'), mmap_mode=mmap_mode)
        else:
            adj_dst_index = None

        def _load_feat(feat_path):
            """Load features from .npy file.
            """
            feat = {}
            if os.path.isdir(feat_path):
                for feat_name in os.listdir(feat_path):
                    feat[os.path.splitext(feat_name)[0]] = np.load(
                        os.path.join(feat_path, feat_name),
                        mmap_mode=mmap_mode)
            return feat

        node_feat = _load_feat(os.path.join(path, 'node_feat'))
        edge_feat = _load_feat(os.path.join(path, 'edge_feat'))
        return cls(edges=edges,
                   num_nodes=num_nodes,
                   node_feat=node_feat,
                   edge_feat=edge_feat,
                   adj_src_index=adj_src_index,
                   adj_dst_index=adj_dst_index,
                   _num_graph=num_graph,
                   _graph_node_index=graph_node_index,
                   _graph_edge_index=graph_edge_index)

    def is_tensor(self):
        """Return whether the Graph is in paddle.Tensor or numpy format.
        """
        return self._is_tensor

    def _apply_to_tensor(self, key, value, inplace=True):
        if value is None:
            return value

        if key == '_is_tensor':
            # set is_tensor to True
            return True

        if isinstance(value, EdgeIndex):
            value = value.tensor(inplace=inplace)

        elif isinstance(value, dict):
            if inplace:
                for k, v in value.items():
                    value[k] = paddle.to_tensor(v)
            else:
                new_value = {}
                for k, v in value.items():
                    new_value[k] = paddle.to_tensor(v)
                value = new_value
        else:
            value = paddle.to_tensor(value)
        return value

    def tensor(self, inplace=True):
        """Convert the Graph into paddle.Tensor format.

        In paddle.Tensor format, the graph edges and node features are in paddle.Tensor format.
        You can use send and recv in paddle.Tensor graph.

        Args:

            inplace: (Default True) Whether to convert the graph into tensor inplace.

        """

        if self._is_tensor:
            return self

        if inplace:
            for key in self.__dict__:
                self.__dict__[key] = self._apply_to_tensor(
                    key, self.__dict__[key], inplace)
            return self
        else:
            new_dict = {}
            for key in self.__dict__:
                new_dict[key] = self._apply_to_tensor(key, self.__dict__[key],
                                                      inplace)

            graph = self.__class__(
                num_nodes=new_dict["_num_nodes"],
                edges=new_dict["_edges"],
                node_feat=new_dict["_node_feat"],
                edge_feat=new_dict["_edge_feat"],
                adj_src_index=new_dict["_adj_src_index"],
                adj_dst_index=new_dict["_adj_dst_index"],
                **new_dict)
            return graph

    def _apply_to_numpy(self, key, value, inplace=True):
        if value is None:
            return value

        if key == '_is_tensor':
            # set is_tensor to True
            return False

        if isinstance(value, EdgeIndex):
            value = value.numpy(inplace=inplace)
        elif isinstance(value, dict):
            if inplace:
                for k, v in value.items():
                    value[k] = v.numpy()
            else:
                new_value = {}
                for k, v in value.items():
                    new_value[k] = v.numpy()
                value = new_value
        else:
            value = value.numpy()
        return value

    def numpy(self, inplace=True):
        """Convert the Graph into numpy format.

        In numpy format, the graph edges and node features are in numpy.ndarray format.
        But you can't use send and recv in numpy graph.

        Args:

            inplace: (Default True) Whether to convert the graph into numpy inplace.

        """
        if not self._is_tensor:
            return self

        if inplace:
            for key in self.__dict__:
                self.__dict__[key] = self._apply_to_numpy(
                    key, self.__dict__[key], inplace)
            return self
        else:
            new_dict = {}
            for key in self.__dict__:
                new_dict[key] = self._apply_to_numpy(key, self.__dict__[key],
                                                     inplace)

            graph = self.__class__(
                num_nodes=new_dict["_num_nodes"],
                edges=new_dict["_edges"],
                node_feat=new_dict["_node_feat"],
                edge_feat=new_dict["_edge_feat"],
                adj_src_index=new_dict["_adj_src_index"],
                adj_dst_index=new_dict["_adj_dst_index"],
                **new_dict)
            return graph

    def dump(self, path):
        """Dump the graph into a directory.

        This function will dump the graph information into the given directory path.
        The graph can be read back with :code:`pgl.Graph.load`

        Args:
            path: The directory for the storage of the graph.

        """
        if self._is_tensor:
            # Convert back into numpy and dump.
            graph = self.numpy(inplace=False)
            graph.dump(path)
        else:
            if not os.path.exists(path):
                os.makedirs(path)

            np.save(os.path.join(path, 'num_nodes.npy'), self._num_nodes)
            np.save(os.path.join(path, 'edges.npy'), self._edges)
            np.save(os.path.join(path, 'num_graph.npy'), self._num_graph)

            if self._adj_src_index is not None:
                self._adj_src_index.dump(os.path.join(path, 'adj_src'))

            if self._adj_dst_index is not None:
                self._adj_dst_index.dump(os.path.join(path, 'adj_dst'))

            if self._graph_node_index is not None:
                np.save(
                    os.path.join(path, 'graph_node_index.npy'),
                    self._graph_node_index)

            if self._graph_edge_index is not None:
                np.save(
                    os.path.join(path, 'graph_edge_index.npy'),
                    self._graph_edge_index)

            def _dump_feat(feat_path, feat):
                """Dump all features to .npy file.
                """
                if len(feat) == 0:
                    return

                if not os.path.exists(feat_path):
                    os.makedirs(feat_path)

                for key in feat:
                    value = feat[key]
                    np.save(os.path.join(feat_path, key + ".npy"), value)

            _dump_feat(os.path.join(path, "node_feat"), self.node_feat)
            _dump_feat(os.path.join(path, "edge_feat"), self.edge_feat)

    @property
    def adj_src_index(self):
        """Return an EdgeIndex object for src.
        """
        if self._adj_src_index is None:
            u = self._edges[:, 0]
            v = self._edges[:, 1]
            self._adj_src_index = EdgeIndex.from_edges(
                u=u, v=v, num_nodes=self._num_nodes)
        return self._adj_src_index

    @property
    def adj_dst_index(self):
        """Return an EdgeIndex object for dst.
        """
        if self._adj_dst_index is None:
            v = self._edges[:, 0]
            u = self._edges[:, 1]
            self._adj_dst_index = EdgeIndex.from_edges(
                u=u, v=v, num_nodes=self._num_nodes)
        return self._adj_dst_index

    @property
    def edge_feat(self):
        """Return a dictionary of edge features.
        """
        return self._edge_feat

    @property
    def node_feat(self):
        """Return a dictionary of node features.
        """
        return self._node_feat

    @property
    def num_edges(self):
        """Return the number of edges.
        """
        if self._is_tensor:
            return paddle.shape(self._edges)[0]
        else:
            return self._edges.shape[0]

    @property
    def num_nodes(self):
        """Return the number of nodes.
        """
        return self._num_nodes

    @property
    def edges(self):
        """Return all edges in numpy.ndarray or paddle.Tensor with shape (num_edges, 2).
        """
        return self._edges

    def sorted_edges(self, sort_by="src"):
        """Return sorted edges with different strategies.

        This function will return sorted edges with different strategy.
        If :code:`sort_by="src"`, then edges will be sorted by :code:`src`
        nodes and otherwise :code:`dst`.

        Args:

            sort_by: The type for sorted edges. ("src" or "dst")

        Return:

            A tuple of (sorted_src, sorted_dst, sorted_eid).
        """
        if sort_by not in ["src", "dst"]:
            raise ValueError("sort_by should be in 'src' or 'dst'.")
        if sort_by == 'src':
            src, dst, eid = self.adj_src_index.triples()
        else:
            dst, src, eid = self.adj_dst_index.triples()
        return src, dst, eid

    @property
    def nodes(self):
        """Return all nodes id from 0 to :code:`num_nodes - 1`
        """
        if self._nodes is None:
            if self.is_tensor():
                self._nodes = paddle.arange(self.num_nodes)
            else:
                self._nodes = np.arange(self.num_nodes)
        return self._nodes

    def indegree(self, nodes=None):
        """Return the indegree of the given nodes

        This function will return indegree of given nodes.

        Args:

            nodes: Return the indegree of given nodes,
                   if nodes is None, return indegree for all nodes

        Return:

            A numpy.ndarray or paddle.Tensor as the given nodes' indegree.
        """
        if nodes is None:
            return self.adj_dst_index.degree
        else:
            if self._is_tensor:
                return paddle.gather(self.adj_dst_index.degree, nodes)
            else:
                return self.adj_dst_index.degree[nodes]

    def outdegree(self, nodes=None):
        """Return the outdegree of the given nodes.

        This function will return outdegree of given nodes.

        Args:

            nodes: Return the outdegree of given nodes,
                   if nodes is None, return outdegree for all nodes

        Return:

            A numpy.array or paddle.Tensor as the given nodes' outdegree.
        """
        if nodes is None:
            return self.adj_src_index.degree
        else:
            if self._is_tensor:
                return paddle.gather(self.adj_src_index.degree, nodes)
            else:
                return self.adj_src_index.degree[nodes]

    def successor(self, nodes=None, return_eids=False):
        """Find successor of given nodes.

        This function will return the successor of given nodes.

        Args:

            nodes: Return the successor of given nodes,
                   if nodes is None, return successor for all nodes.

            return_eids: If True return nodes together with corresponding eid

        Return:

            Return a list of numpy.ndarray and each numpy.ndarray represent a list
            of successor ids for given nodes. If :code:`return_eids=True`, there will
            be an additional list of numpy.ndarray and each numpy.ndarray represent
            a list of eids that connected nodes to their successors.

        Example:
            .. code-block:: python

                import numpy as np
                import pgl

                num_nodes = 5
                edges = [ (0, 1), (1, 2), (3, 4)]
                graph = pgl.Graph(num_nodes=num_nodes,
                        edges=edges)
                succ, succ_eid = graph.successor(return_eids=True)

            This will give output.

            .. code-block:: python

                succ:
                      [[1],
                       [2],
                       [],
                       [4],
                       []]

                succ_eid:
                      [[0],
                       [1],
                       [],
                       [2],
                       []]

        """
        if self.is_tensor():
            raise ValueError(
                "You must call Graph.numpy() first. Tensor object don't supprt successor now."
            )
        else:
            if return_eids:
                return self.adj_src_index.view_v(
                    nodes), self.adj_src_index.view_eid(nodes)
            else:
                return self.adj_src_index.view_v(nodes)

    def sample_successor(self,
                         nodes,
                         max_degree,
                         return_eids=False,
                         shuffle=False):
        """Sample successors of given nodes.

        Args:

            nodes: Given nodes whose successors will be sampled.

            max_degree: The max sampled successors for each nodes.

            return_eids: Whether to return the corresponding eids.

        Return:

            Return a list of numpy.ndarray and each numpy.ndarray represent a list
            of sampled successor ids for given nodes. If :code:`return_eids=True`, there will
            be an additional list of numpy.ndarray and each numpy.ndarray represent
            a list of eids that connected nodes to their successors.
        """
        if self.is_tensor():
            raise ValueError(
                "You must call Graph.numpy() first. Tensor object don't supprt sample_successor now."
            )
        else:
            node_succ = self.successor(nodes, return_eids=return_eids)
            if return_eids:
                node_succ, node_succ_eid = node_succ

            if nodes is None:
                nodes = self.nodes

            node_succ = node_succ.tolist()

            if return_eids:
                node_succ_eid = node_succ_eid.tolist()

            if return_eids:
                return graph_kernel.sample_subset_with_eid(
                    node_succ, node_succ_eid, max_degree, shuffle)
            else:
                return graph_kernel.sample_subset(node_succ, max_degree,
                                                  shuffle)

    def predecessor(self, nodes=None, return_eids=False):
        """Find predecessor of given nodes.

        This function will return the predecessor of given nodes.

        Args:

            nodes: Return the predecessor of given nodes,
                   if nodes is None, return predecessor for all nodes.

            return_eids: If True return nodes together with corresponding eid

        Return:

            Return a list of numpy.ndarray and each numpy.ndarray represent a list
            of predecessor ids for given nodes. If :code:`return_eids=True`, there will
            be an additional list of numpy.ndarray and each numpy.ndarray represent
            a list of eids that connected nodes to their predecessors.

        Example:

            .. code-block:: python

                import numpy as np
                import pgl

                num_nodes = 5
                edges = [ (0, 1), (1, 2), (3, 4)]
                graph = pgl.Graph(num_nodes=num_nodes,
                        edges=edges)
                pred, pred_eid = graph.predecessor(return_eids=True)

            This will give output.

            .. code-block:: python

                pred:
                      [[],
                       [0],
                       [1],
                       [],
                       [3]]

                pred_eid:
                      [[],
                       [0],
                       [1],
                       [],
                       [2]]

        """
        if self.is_tensor():
            raise ValueError(
                "You must call Graph.numpy() first. Tensor object don't supprt predecessor now."
            )
        else:
            if return_eids:
                return self.adj_dst_index.view_v(
                    nodes), self.adj_dst_index.view_eid(nodes)
            else:
                return self.adj_dst_index.view_v(nodes)

    def sample_predecessor(self,
                           nodes,
                           max_degree,
                           return_eids=False,
                           shuffle=False):
        """Sample predecessor of given nodes.

        Args:

            nodes: Given nodes whose predecessor will be sampled.

            max_degree: The max sampled predecessor for each nodes.

            return_eids: Whether to return the corresponding eids.

        Return:

            Return a list of numpy.ndarray and each numpy.ndarray represent a list
            of sampled predecessor ids for given nodes. If :code:`return_eids=True`, there will
            be an additional list of numpy.ndarray and each numpy.ndarray represent
            a list of eids that connected nodes to their predecessors.
        """
        if self.is_tensor():
            raise ValueError(
                "You must call Graph.numpy() first. Tensor object don't supprt sample_predecessor now."
            )
        else:
            node_pred = self.predecessor(nodes, return_eids=return_eids)
            if return_eids:
                node_pred, node_pred_eid = node_pred

            if nodes is None:
                nodes = self.nodes

            node_pred = node_pred.tolist()

            if return_eids:
                node_pred_eid = node_pred_eid.tolist()

            if return_eids:
                return graph_kernel.sample_subset_with_eid(
                    node_pred, node_pred_eid, max_degree, shuffle)
            else:
                return graph_kernel.sample_subset(node_pred, max_degree,
                                                  shuffle)

    @property
    def num_graph(self):
        """ Return Number of Graphs"""
        return self._num_graph

    @property
    def graph_node_id(self):
        """ Return a numpy.ndarray or paddle.Tensor with shape [num_nodes]
        that indicates which graph the nodes belongs to.

        Examples:

        .. code-block:: python

            import numpy as np
            import pgl

            num_nodes = 5
            edges = [ (0, 1), (1, 2), (3, 4)]
            graph = pgl.Graph(num_nodes=num_nodes,
                        edges=edges)
            joint_graph = pgl.Graph.batch([graph, graph])
            print(joint_graph.graph_node_id)

            >>> [0, 0, 0, 0, 0, 1, 1, 1, 1 ,1]

        """

        return generate_segment_id_from_index(self._graph_node_index)

    @property
    def graph_edge_id(self):
        """ Return a numpy.ndarray or paddle.Tensor with shape [num_edges]
        that indicates which graph the edges belongs to.

        Examples:

        .. code-block:: python

            import numpy as np
            import pgl

            num_nodes = 5
            edges = [ (0, 1), (1, 2), (3, 4)]
            graph = pgl.Graph(num_nodes=num_nodes,
                        edges=edges)
            joint_graph = pgl.Graph.batch([graph, graph])
            print(joint_graph.graph_edge_id)

            >>> [0, 0, 0, 1, 1, 1]

        """

        return generate_segment_id_from_index(self._graph_edge_index)

    def send_recv(self, feature, reduce_func="sum"):
        """This method combines the send and recv function using graph_send_recv API.

        Now, this method only supports default copy send function, and built-in receive
        function ('sum', 'mean', 'max', 'min').

        Args:

           feature (Tensor): The node feature of a graph.

           reduce_func (str): Difference reduce function, including 'sum', 'mean', 'max', 'min'.

        """

        assert isinstance(feature, paddle.Tensor) or isinstance(feature, paddle.fluid.framework.Variable), \
            "The input of send_recv method should be Tensor."

        assert reduce_func in ['sum', 'mean', 'max', 'min'], \
            "Only support 'sum', 'mean', 'max', 'min' built-in reduce functions."

        src, dst = self.edges[:, 0], self.edges[:, 1]
        return graph_send_recv(feature, src, dst, pool_type=reduce_func)

    def send(
            self,
            message_func,
            src_feat=None,
            dst_feat=None,
            edge_feat=None,
            node_feat=None, ):
        """Send message from all src nodes to dst nodes.

        The UDF message function should has the following format.

        .. code-block:: python

            def message_func(src_feat, dst_feat, edge_feat):
                '''
                    Args:
                        src_feat: the node feat dict attached to the src nodes.
                        dst_feat: the node feat dict attached to the dst nodes.
                        edge_feat: the edge feat dict attached to the
                                   corresponding (src, dst) edges.

                    Return:
                        It should return a tensor or a dictionary of tensor. And each tensor
                        should have a shape of (num_edges, dims).
                '''
                return {'msg': src_feat['h']}

        Args:
            message_func: UDF function.
            src_feat: a dict {name: tensor,} to build src node feat
            dst_feat: a dict {name: tensor,} to build dst node feat
            node_feat: a dict {name: tensor,} to build both src and dst node feat
            edge_feat: a dict {name: tensor,} to build edge feat

        Return:
            A dictionary of tensor representing the message. Each of the values
            in the dictionary has a shape (num_edges, dim) which should be collected
            by :code:`recv` function.
        """
        if self._is_tensor:
            if (src_feat is not None or
                    dst_feat is not None) and node_feat is not None:
                raise ValueError(
                    "Can not use src/dst feat and node feat at the same time")

            src_feat_temp = {}
            dst_feat_temp = {}
            if node_feat is not None:
                assert isinstance(node_feat,
                                  dict), "The input node_feat must be a dict"
                src_feat_temp.update(node_feat)
                dst_feat_temp.update(node_feat)
            else:
                if src_feat is not None:
                    assert isinstance(
                        src_feat, dict), "The input src_feat must be a dict"
                    src_feat_temp.update(src_feat)

                if dst_feat is not None:
                    assert isinstance(
                        dst_feat, dict), "The input dst_feat must be a dict"
                    dst_feat_temp.update(dst_feat)

            edge_feat_temp = {}
            if edge_feat is not None:
                assert isinstance(edge_feat,
                                  dict), "The input edge_feat must be a dict"
                edge_feat_temp.update(edge_feat)

            src = self.edges[:, 0]
            dst = self.edges[:, 1]

            src_feat = op.RowReader(src_feat_temp, src)
            dst_feat = op.RowReader(dst_feat_temp, dst)
            msg = message_func(src_feat, dst_feat, edge_feat_temp)

            if not isinstance(msg, dict):
                raise TypeError(
                    "The outputs of the %s function is expected to be a dict, but got %s" \
                            % (message_func.__name__, type(msg)))
            return msg
        else:
            raise ValueError("You must call Graph.tensor() first")

    def _process_graph_info(self, **kwargs):
        if ("_graph_node_index" in kwargs) and (
                kwargs["_graph_node_index"] is not None):
            self._graph_node_index = kwargs["_graph_node_index"]
        else:
            self._graph_node_index = None

        if ("_graph_edge_index" in kwargs) and (
                kwargs["_graph_edge_index"] is not None):
            self._graph_edge_index = kwargs["_graph_edge_index"]
        else:
            self._graph_edge_index = None

        if ("_num_graph" in kwargs) and (kwargs["_num_graph"] is not None):
            self._num_graph = kwargs["_num_graph"]
        else:
            if self._is_tensor:
                self._num_graph = paddle.ones(shape=[1], dtype="int32")
                self._graph_node_index = paddle.concat([
                    paddle.zeros(
                        shape=[1], dtype="int32"), paddle.full(
                            shape=[1],
                            fill_value=self.num_nodes,
                            dtype="int32")
                ])
                self._graph_edge_index = paddle.concat([
                    paddle.zeros(
                        shape=[1], dtype="int32"), paddle.full(
                            shape=[1],
                            fill_value=self.num_edges,
                            dtype="int32")
                ])
            else:
                self._num_graph = 1
                self._graph_node_index = np.array(
                    [0, self._num_nodes], dtype="int64")
                self._graph_edge_index = np.array(
                    [0, self.num_edges], dtype="int64")

    @classmethod
    def disjoint(cls, graph_list, merged_graph_index=False):
        """This method disjoint list of graph into a big graph.

        Args:

            graph_list (Graph List): A list of Graphs.

            merged_graph_index: whether to keeped the graph_id that the nodes belongs to.

        .. code-block:: python

            import numpy as np
            import pgl

            num_nodes = 5
            edges = [ (0, 1), (1, 2), (3, 4)]
            graph = pgl.Graph(num_nodes=num_nodes,
                        edges=edges)
            joint_graph = pgl.Graph.disjoint([graph, graph], merged_graph_index=False)
            print(joint_graph.graph_node_id)
            >>> [0, 0, 0, 0, 0, 1, 1, 1, 1 ,1]
            print(joint_graph.num_graph)
            >>> 2

            joint_graph = pgl.Graph.disjoint([graph, graph], merged_graph_index=True)
            print(joint_graph.graph_node_id)
            >>> [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
            print(joint_graph.num_graph)
            >>> 1
        """
        # TODO:@Yelrose supporting disjoint a disjointed graph_list.
        assert len(
            graph_list
        ) > 0, "The input graph_list of Graph.disjoint has length %s. It should be greater than 0. " % len(
            graph_list)

        is_tensor = graph_list[0].is_tensor()

        edges = cls._join_edges(graph_list)
        num_nodes = cls._join_nodes(graph_list)
        node_feat = cls._join_feature(graph_list, mode="node")
        edge_feat = cls._join_feature(graph_list, mode="edge")

        if merged_graph_index is True:
            num_graph = None
            graph_node_index = None
            graph_edge_index = None
        else:
            num_graph = paddle.to_tensor([len(graph_list)], "int64") \
                    if is_tensor else len(graph_list)
            graph_node_index = cls._join_graph_index(graph_list, mode="node")
            graph_edge_index = cls._join_graph_index(graph_list, mode="edge")

        graph = cls(num_nodes=num_nodes,
                    edges=edges,
                    node_feat=node_feat,
                    edge_feat=edge_feat,
                    _num_graph=num_graph,
                    _graph_node_index=graph_node_index,
                    _graph_edge_index=graph_edge_index)
        return graph

    @staticmethod
    def batch(graph_list):
        """This is alias on `pgl.Graph.disjoint` with merged_graph_index=False"""
        return Graph.disjoint(graph_list, merged_graph_index=False)

    @staticmethod
    def _join_graph_index(graph_list, mode="node"):
        is_tensor = graph_list[0].is_tensor()
        if mode == "node":
            counts = [g.num_nodes for g in graph_list]
        elif mode == "edge":
            counts = [g.num_edges for g in graph_list]
        else:
            raise ValueError(
                "mode must be in ['node', 'edge']. But received model=%s" %
                mode)

        if is_tensor:
            counts = paddle.concat(counts)
        return op.get_index_from_counts(counts)

    @staticmethod
    def _join_nodes(graph_list):
        num_nodes = 0
        for g in graph_list:
            num_nodes = g.num_nodes + num_nodes
        return num_nodes

    @staticmethod
    def _join_feature(graph_list, mode="node"):
        """join node features for multiple graph"""
        is_tensor = graph_list[0].is_tensor()
        feat = defaultdict(lambda: [])
        if mode == "node":
            for graph in graph_list:
                for key in graph.node_feat:
                    feat[key].append(graph.node_feat[key])
        elif mode == "edge":
            for graph in graph_list:
                for key in graph.edge_feat:
                    feat[key].append(graph.edge_feat[key])
        else:
            raise ValueError(
                "mode must be in ['node', 'edge']. But received model=%s" %
                mode)

        ret_feat = {}
        for key in feat:
            if len(feat[key]) == 1:
                ret_feat[key] = feat[key][0]
            else:
                if is_tensor:
                    ret_feat[key] = paddle.concat(feat[key], 0)
                else:
                    ret_feat[key] = np.concatenate(feat[key], axis=0)
        return ret_feat

    @staticmethod
    def _join_edges(graph_list):
        """join edges for multiple graph"""
        is_tensor = graph_list[0].is_tensor()
        list_edges = []
        start_offset = 0
        for graph in graph_list:
            edges = graph.edges
            if len(edges) > 0:
                edges = edges + start_offset
                list_edges.append(edges)
            start_offset += graph.num_nodes
        if len(list_edges) == 1:
            return list_edges[0]

        if is_tensor:
            edges = paddle.concat(list_edges, 0)
        else:
            edges = np.concatenate(list_edges, axis=0)
        return edges

    def node_batch_iter(self, batch_size, shuffle=True):
        """Node batch iterator

        Iterate all node by batch.

        Args:
            batch_size: The batch size of each batch of nodes.

            shuffle: Whether shuffle the nodes.

        Return:
            Batch iterator
        """
        if self.is_tensor():
            if shuffle:
                perm = paddle.randperm(self.num_nodes)
            else:
                perm = paddle.arange(self.num_nodes)
        else:
            perm = np.arange(self.num_nodes)
            if shuffle:
                np.random.shuffle(perm)

        start = 0
        while start < self.num_nodes:
            yield perm[start:start + batch_size]
            start += batch_size

    def to_mmap(self, path="./tmp"):
        """Turn the Graph into Memmap mode which can share memory between processes.
        """
        self.dump(path)
        graph = Graph.load(path, mmap_mode="r")
        return graph

class DistGPUGraph(Graph):
    """Implement of Distributed GPU Graph.

    DistGPUGraph provide ability to shard a graph into MultiGPU to
    allow MultiGPU FullBatch Training with little changes of codes.

    Currenlty, we only share edges into multiple GPU to reduce the time
    and memory usage when applying functions on edges. When apply DistGraph
    on Graph, edges and edge features will be share into different GPU. But
    node features still have its own copies across all GPUs. And we utilize
    NCCL allreduce to make sure all the node features are consistence between
    GPU cards.

    THIS IS A EXPERIMENTAL API.

    Example:

       .. code-block:: python

            import numpy as np
            import pgl
            import paddle.distributed as dist

            # init distributed env
            dist.init_parallel_env()

            num_nodes = 5
            edges = [ (0, 1), (1, 2), (3, 4)]
            feature = np.random.randn(5, 100).astype(np.float32)
            edge_feature = np.random.randn(3, 100).astype(np.float32)

            graph = pgl.Graph(num_nodes=num_nodes,
                        edges=edges,
                        node_feat={
                            "feature": feature
                        },
                        edge_feat={
                            "edge_feature": edge_feature
                        })

            # sharding
            graph = pgl.DistGraph(graph)

            model = pgl.nn.GCNConv(100, 100)
            model = paddle.DataParallel(model)

            out = model(graph, graph.node_feat["feature"])

    """

    def __init__(self, graph):
        warnings.warn("DistGPUGraph is an experimental API"
                      " for Multi-GPU FullBatch Training.")
        shard_edges, shard_edge_feat = self._shard_edges_by_dst(
            graph.edges, graph.edge_feat)
        super(DistGPUGraph, self).__init__(
            num_nodes=graph.num_nodes,
            edges=shard_edges,
            node_feat=graph.node_feat,
            edge_feat=shard_edge_feat)

        if not self.is_tensor():
            self.tensor(inplace=True)

    def _shard_edges_by_dst(self, edges, edge_feat):
        """Shard Edges by dst

        Args:

            edges: list of (u, v) tuples, 2D numpy.ndarry or 2D paddle.Tensor

            edge_feat (optional): a dict of numpy array as edge features (should
                                have consistent order with edges)

        Returns:

            Return a tuple (shard_edges, shard_edge_feat) as the shard results.

        """
        shard_flag = edges[:, 1]
        mask = (shard_flag % dist.get_world_size()) == dist.get_rank()
        if type(mask) == paddle.Tensor:
            eid = paddle.masked_select(paddle.arange(edges.shape[0]), mask)
            shard_edges = paddle.gather(edges, eid)
            shard_edge_feat = {}
            for key, value in edge_feat.items():
                shard_edge_feat[key] = paddle.gather(value, eid)
        else:
            eid = np.arange(edges.shape[0])[mask]
            shard_edges = edges[eid]
            shard_edge_feat = {}
            for key, value in edge_feat.items():
                shard_edge_feat[key] = value[eid]
        return shard_edges, shard_edge_feat

    def numpy(self, inplace=True):
        raise ValueError("DistGPUGraph can't convert into numpy")

    def recv(self, reduce_func, msg, recv_mode="dst"):
        if recv_mode != "dst":
            raise ValueError(
                "Currently DistGPUGraph can only support recv_mode=='dst'")
        output = super(DistGPUGraph, self).recv(
            msg=msg, reduce_func=reduce_func, recv_mode=recv_mode)
        out = op.all_reduce_sum_with_grad(output)
        return out

    def indegree(self, nodes=None):
        degree = super(DistGPUGraph, self).indegree(nodes=nodes)
        degree = op.all_reduce_sum_with_grad(degree)
        return degree

    def outdegree(self, nodes=None):
        degree = super(DistGPUGraph, self).outdegree(nodes=nodes)
        degree = op.all_reduce_sum_with_grad(degree)
        return degree

    def send_recv(self, feature, reduce_func="sum"):
        output = super(DistGPUGraph, self).send_recv(
            feature=feature, reduce_func="sum")
        output = op.all_reduce_sum_with_grad(output)
        return output

max_stars_repo_name: zbmain/PGL

max_issues_repo_name: zbmain/PGL

max_forks_repo_name: zbmain/PGL


text:
import open3d as o3d
import sys
import numpy as np
import pandas as pd
import csv

#check how many arguments
if(len(sys.argv) == 5):
	print('Correct number of arguments')

'''
r = int(34)
g = int(200)
b = int(78)

rgb = (r << 16) | (g << 8) | b

print(str(rgb))

r_back = (rgb >> 16) & 0x0000ff;
g_back  = (rgb >> 8)  & 0x0000ff;
b_back  = (rgb)       & 0x0000ff;

print('\n')
print('\n')
print(str(r_back))
print('\n')
print(str(g_back))
print('\n')
print(str(b_back))
print('\n')
'''

#Get inputs
pathToCsv = sys.argv[1]
height = int(sys.argv[2])
width = int(sys.argv[3])
depths = int(sys.argv[4])

#Create a pcd file
pcdFilename = 'heyhey.pcd'

pcdFile = open(pcdFilename,'w')

pcdFile.write('VERSION .7\nFIELDS x y z rgb\nSIZE 4 4 4 4\nTYPE I I I F\nCOUNT 1 1 1 1\n')
pcdFile.write('WIDTH ' + str(width) + '\n')
pcdFile.write('HEIGHT ' + str(height) + '\n')
pcdFile.write('VIEWPOINT 0 0 0 1 0 0 0\n')
pcdFile.write('POINTS ' + str(width*height) + '\n')
pcdFile.write('DATA ascii\n')

#print('hello')

with open(pathToCsv) as csvfile:
	readCSV = csv.reader(csvfile, delimiter=',')
	for row in readCSV:
		#read from csv

		x = int(row[0]);
		y = int(row[1]);
		z = int(row[2]);

		r = int(row[3]);
		g = int(row[4]);
		b = int(row[5]);
		'''
		x = 53
		y= 42
		z = 245
		r = 82
		g = 3
		b = 0
		'''
		rgb = (r << 16) | (g << 8) | b

		#write to pcd file
		pcdFile.write(str(x) + ' ' + str(y) + ' ' + str(z) + ' ' + str(rgb) + '\n')

#Close pcd file
pcdFile.close()

#visualize
pcd = o3d.io.read_point_cloud("ex.pcd")
#print(pcd)
#print(np.asarray(pcd.points))
o3d.visualization.draw_geometries([pcd])

ext: py

reason: import numpy


text:
import tkinter as tk
import tkinter.font
import numpy as np
import math
import platform
from ._manager import manager

class ViewerTab():
    """Base class for viewer tabs.

    You must also call viewer.register in the __init__ function.

    You must declare on_destroy(self): This method is called by Viewer when the tab is closed by the user. It is not called if the whole window is closed. It must call viewer.unregister(self).
    """

    def _on_key(self, event):
        """
        Called for keyboard events. Can be overriden by subclasses if needed
        """
        pass

    def _calc_font(self, user_font_size):
        """
        sets self.cell_font and self.font_size according to the specified user font and the screen DPI
        """
        dpi = manager.get_or_create_root().winfo_fpixels('1i')
        if user_font_size is None:
            if dpi >= 200:
                self.font_size = 28
            elif dpi >= 150:
                self.font_size = 22
            elif dpi >= 90:
                self.font_size = 16
            else:
                self.font_size = 14
        else:
            self.font_size = user_font_size

        # default root window needed to create font. -size -> size in pixels instead of inches
        if platform.system() == 'Linux':
            self.cell_font = tk.font.Font(size=-self.font_size, family="Sans")  # Arial was messing up
        else:
            self.cell_font = tk.font.Font(size=-self.font_size, family="Arial")


text:
"""
Module for wrapping many Op parameters into one object available in both Python and C code.

The module provides the main public class :class:`ParamsType` that allows to bundle many Aesara types
into one parameter type, and an internal convenient class :class:`Params` which will be automatically
used to create a Params object that is compatible with the ParamsType defined.

The Params object will be available in both Python code (as a standard Python object) and C code
(as a specific struct with parameters as struct fields). To be fully-available in C code, Aesara
types wrapped into a ParamsType must provide a C interface (e.g. TensorType, Scalar, GpuArrayType,
or your own type. See :ref:`extending_op_params` for more details).

Example of usage
----------------

Importation:

.. code-block:: python

    # Import ParamsType class.
    from aesara.graph.params_type import ParamsType

    # If you want to use a tensor and a scalar as parameters,
    # you should import required Aesara types.
    from aesara.tensor.type import TensorType
    from aesara.scalar import Scalar

In your Op sub-class:

.. code-block:: python

    params_type = ParamsType(attr1=TensorType('int32', (False, False)), attr2=Scalar('float64'))

If your op contains attributes ``attr1`` **and** ``attr2``, the default ``op.get_params()``
implementation will automatically try to look for it and generate an appropriate Params object.
Attributes must be compatible with the corresponding types defined into the ParamsType
(we will try to convert and downcast if needed). In this example, ``your_op.attr1``
should be a matrix of integers, and ``your_op.attr2``
should be a real number (integer or floating value).

.. code-block:: python

    def __init__(value_attr1, value_attr2):
        self.attr1 = value_attr1
        self.attr2 = value_attr2

In ``perform()`` implementation (with params named ``param``):

.. code-block:: python

    matrix_param = param.attr1
    number_param = param.attr2

In ``c_code()`` implementation (with ``param = sub['params']``):

.. code-block:: c

    PyArrayObject* matrix = param->attr1;
    npy_float64    number = param->attr2;
    /* You won't need to free them or whatever else. */

See :class:`QuadraticOpFunc` and :class:`QuadraticCOpFunc` in ``aesara/graph/tests/test_params_type.py``
for complete working examples.

Combining ParamsType with Aesara enumeration types
--------------------------------------------------

Aesara provide some enumeration types that allow to create constant primitive values (integer and floating values)
available in both Python and C code. See :class:`aesara.graph.type.EnumType` and its subclasses for more details.

If your ParamsType contains Aesara enumeration types, then constants defined inside these
enumerations will be directly available as ParamsType attributes.

**Example**::

    from aesara.graph.params_type import ParamsType
    from aesara.graph.type import EnumType, EnumList

    wrapper = ParamsType(enum1=EnumList('CONSTANT_1', 'CONSTANT_2', 'CONSTANT_3'),
                         enum2=EnumType(PI=3.14, EPSILON=0.001))

    # Each enum constant is available as a wrapper attribute:
    print(wrapper.CONSTANT_1, wrapper.CONSTANT_2, wrapper.CONSTANT_3,
          wrapper.PI, wrapper.EPSILON)

    # For convenience, you can also look for a constant by name with
    # ``ParamsType.get_enum()`` method.
    pi = wrapper.get_enum('PI')
    epsilon = wrapper.get_enum('EPSILON')
    constant_2 = wrapper.get_enum('CONSTANT_2')
    print(pi, epsilon, constant_2)

This implies that a ParamsType cannot contain different enum types with common enum names::

    # Following line will raise an error,
    # as there is a "CONSTANT_1" defined both in enum1 and enum2.
    wrapper = ParamsType(enum1=EnumList('CONSTANT_1', 'CONSTANT_2'),
                         enum2=EnumType(CONSTANT_1=0, CONSTANT_3=5))

If your enum types contain constant aliases, you can retrieve them from ParamsType
with ``ParamsType.enum_from_alias(alias)`` method (see :class:`aesara.graph.type.EnumType`
for more info about enumeration aliases).

.. code-block:: python

    wrapper = ParamsType(enum1=EnumList('A', ('B', 'beta'), 'C'),
                         enum2=EnumList(('D', 'delta'), 'E', 'F'))
    b1 = wrapper.B
    b2 = wrapper.get_enum('B')
    b3 = wrapper.enum_from_alias('beta')
    assert b1 == b2 == b3

"""

import hashlib
import re

from aesara.graph.type import CType, EnumType
from aesara.graph.utils import MethodNotDefined

# Set of C and C++ keywords as defined (at March 2nd, 2017) in the pages below:
# - http://fr.cppreference.com/w/c/keyword
# - http://fr.cppreference.com/w/cpp/keyword
# Added `NULL` and `_Pragma` keywords.
c_cpp_keywords = {
    "_Alignas",
    "_Alignof",
    "_Atomic",
    "_Bool",
    "_Complex",
    "_Generic",
    "_Imaginary",
    "_Noreturn",
    "_Pragma",
    "_Static_assert",
    "_Thread_local",
    "alignas",
    "alignof",
    "and",
    "and_eq",
    "asm",
    "auto",
    "bitand",
    "bitor",
    "bool",
    "break",
    "case",
    "catch",
    "char",
    "char16_t",
    "char32_t",
    "class",
    "compl",
    "const",
    "const_cast",
    "constexpr",
    "continue",
    "decltype",
    "default",
    "delete",
    "do",
    "double",
    "dynamic_cast",
    "else",
    "enum",
    "explicit",
    "export",
    "extern",
    "false",
    "float",
    "for",
    "friend",
    "goto",
    "if",
    "inline",
    "int",
    "long",
    "mutable",
    "namespace",
    "new",
    "noexcept",
    "not",
    "not_eq",
    "NULL",
    "nullptr",
    "operator",
    "or",
    "or_eq",
    "private",
    "protected",
    "public",
    "register",
    "reinterpret_cast",
    "restrict",
    "return",
    "short",
    "signed",
    "sizeof",
    "static",
    "static_assert",
    "static_cast",
    "struct",
    "switch",
    "template",
    "this",
    "thread_local",
    "throw",
    "true",
    "try",
    "typedef",
    "typeid",
    "typename",
    "union",
    "unsigned",
    "using",
    "virtual",
    "void",
    "volatile",
    "wchar_t",
    "while",
    "xor",
    "xor_eq",
}

class Params(dict):
    """
    Internal convenient class to wrap many Python objects into one
    (this class is not safe as the hash method does not check if values are effectively hashable).

    **Example:**

    .. code-block:: python

        from aesara.graph.params_type import ParamsType, Params
        from aesara.scalar import Scalar
        # You must create a ParamsType first:
        params_type = ParamsType(attr1=Scalar('int32'),
                                 key2=Scalar('float32'),
                                 field3=Scalar('int64'))
        # Then you can create a Params object with
        # the params type defined above and values for attributes.
        params = Params(params_type, attr1=1, key2=2.0, field3=3)
        print(params.attr1, params.key2, params.field3)
        d = dict(attr1=1, key2=2.5, field3=-1)
        params2 = Params(params_type, **d)
        print(params2.attr1, params2.key2, params2.field3)

    """

    def __init__(self, params_type, **kwargs):
        if not isinstance(params_type, ParamsType):
            raise TypeError("Params: 1st constructor argument should be a ParamsType.")
        for field in params_type.fields:
            if field not in kwargs:
                raise TypeError(
                    f'Params: ParamsType attribute "{field}" not in Params args.'
                )
        super().__init__(**kwargs)
        self.__dict__.update(__params_type__=params_type, __signatures__=None)

    def __repr__(self):
        return "Params(%s)" % ", ".join(
            [
                ("{}:{}:{}".format(k, type(self[k]).__name__, self[k]))
                for k in sorted(self.keys())
            ]
        )

    def __getattr__(self, key):
        if key not in self:
            raise AttributeError(f'Params: attribute "{key}" does not exist.')
        return self[key]

    def __setattr__(self, key, value):
        raise NotImplementedError("Params is immutable")

    def __setitem__(self, key, value):
        raise NotImplementedError("Params is immutable")

    def __delitem__(self, key):
        raise NotImplementedError("Params is immutable")

    def __hash__(self):
        # As values are immutable, we can save data signatures the first time
        # to not regenerate them in future hash() calls.
        if self.__signatures__ is None:
            # NB: For writing, we must bypass setattr() which is always called by default by Python.
            self.__dict__["__signatures__"] = tuple(
                # NB: Params object should have been already filtered.
                self.__params_type__.types[i]
                .make_constant(self[self.__params_type__.fields[i]])
                .signature()
                for i in range(self.__params_type__.length)
            )
        return hash((type(self), self.__params_type__) + self.__signatures__)

    def __eq__(self, other):
        return (
            type(self) == type(other)
            and self.__params_type__ == other.__params_type__
            and all(
                # NB: Params object should have been already filtered.
                self.__params_type__.types[i].values_eq(
                    self[self.__params_type__.fields[i]],
                    other[self.__params_type__.fields[i]],
                )
                for i in range(self.__params_type__.length)
            )
        )

    def __ne__(self, other):
        return not self.__eq__(other)

class ParamsType(CType):
    """
    This class can create a struct of Aesara types (like `TensorType`,
    `GpuArrayType`, etc.)  to be used as a convenience op parameter wrapping
    many data.

    `ParamsType` constructor takes key-value args.  Key will be the name of the
    attribute in the struct.  Value is the Aesara type of this attribute,
    ie. an instance of (a subclass of) :class:`CType`
    (eg. ``TensorType('int64', (False,))``).

    In a Python code any attribute named ``key`` will be available via::

        structObject.key

    In a C code, any attribute named ``key`` will be available via:

    .. code-block:: c

        structObject->key;

    .. note::

        This `Type` is not complete and should never be used for regular graph
        operations.

    """

    def __init__(self, **kwargs):
        if len(kwargs) == 0:
            raise ValueError("Cannot create ParamsType from empty data.")

        for attribute_name in kwargs:
            if re.match("^[A-Za-z_][A-Za-z0-9_]*$", attribute_name) is None:
                raise AttributeError(
                    'ParamsType: attribute "%s" should be a valid identifier.'
                    % attribute_name
                )
            if attribute_name in c_cpp_keywords:
                raise SyntaxError(
                    'ParamsType: "%s" is a potential C/C++ keyword and should not be used as attribute name.'
                    % attribute_name
                )
            type_instance = kwargs[attribute_name]
            type_name = type_instance.__class__.__name__
            if not isinstance(type_instance, CType):
                raise TypeError(
                    'ParamsType: attribute "%s" should inherit from Aesara CType, got "%s".'
                    % (attribute_name, type_name)
                )

        self.length = len(kwargs)
        self.fields = tuple(sorted(kwargs.keys()))
        self.types = tuple(kwargs[field] for field in self.fields)
        self.name = self.generate_struct_name()

        self.__const_to_enum = {}
        self.__alias_to_enum = {}
        enum_types = [t for t in self.types if isinstance(t, EnumType)]
        if enum_types:
            # We don't want same enum names in different enum types.
            if sum(len(t) for t in enum_types) != len(
                {k for t in enum_types for k in t}
            ):
                raise AttributeError(
                    "ParamsType: found different enum types with common constants names."
                )
            # We don't want same aliases in different enum types.
            if sum(len(t.aliases) for t in enum_types) != len(
                {alias for t in enum_types for alias in t.aliases}
            ):
                raise AttributeError(
                    "ParamsType: found different enum types with common constants aliases."
                )
            # We don't want aliases that have same names as some constants.
            all_enums = {e for t in enum_types for e in t}
            all_aliases = {a for t in enum_types for a in t.aliases}
            if [a for a in all_aliases if a in all_enums]:
                raise AttributeError(
                    "ParamsType: found aliases that have same names as constants."
                )
            # We map each enum name to the enum type in which it is defined.
            # We will then use this dict to find enum value when looking for enum name in ParamsType object directly.
            self.__const_to_enum = {
                enum_name: enum_type
                for enum_type in enum_types
                for enum_name in enum_type
            }
            self.__alias_to_enum = {
                alias: enum_type
                for enum_type in enum_types
                for alias in enum_type.aliases
            }

    def __setstate__(self, state):
        # NB:
        # I have overridden __getattr__ to make enum constants available through
        # the ParamsType when it contains enum types. To do that, I use some internal
        # attributes: self.__const_to_enum and self.__alias_to_enum. These attributes
        # are normally found by Python without need to call getattr(), but when the
        # ParamsType is unpickled, it seems gettatr() may be called at a point before
        # __const_to_enum or __alias_to_enum are unpickled, so that gettatr() can't find
        # those attributes, and then loop infinitely.
        # For this reason, I must add this trivial implementation of __setstate__()
        # to avoid errors when unpickling.
        self.__dict__.update(state)

    def __getattr__(self, key):
        # Now we can access value of each enum defined inside enum types wrapped into the current ParamsType.
        if key in self.__const_to_enum:
            return self.__const_to_enum[key][key]
        return super().__getattr__(self, key)

    def __repr__(self):
        return "ParamsType<%s>" % ", ".join(
            [
                ("{}:{}".format(self.fields[i], self.types[i]))
                for i in range(self.length)
            ]
        )

    def __eq__(self, other):
        return (
            type(self) == type(other)
            and self.fields == other.fields
            and self.types == other.types
        )

    def __hash__(self):
        return hash((type(self),) + self.fields + self.types)

    def generate_struct_name(self):
        # This method tries to generate an unique name for the current instance.
        # This name is intended to be used as struct name in C code and as constant
        # definition to check if a similar ParamsType has already been created
        # (see c_support_code() below).
        fields_string = ",".join(self.fields).encode("utf-8")
        types_string = ",".join(str(t) for t in self.types).encode("utf-8")
        fields_hex = hashlib.sha256(fields_string).hexdigest()
        types_hex = hashlib.sha256(types_string).hexdigest()
        return f"_Params_{fields_hex}_{types_hex}"

    def has_type(self, aesara_type):
        """
        Return True if current ParamsType contains the specified Aesara type.

        """
        return aesara_type in self.types

    def get_type(self, field_name):
        """
        Return the Aesara type associated to the given field name
        in the current ParamsType.

        """
        return self.types[self.fields.index(field_name)]

    def get_field(self, aesara_type):
        """
        Return the name (string) of the first field associated to
        the given Aesara type. Fields are sorted in lexicographic
        order. Raise an exception if this Aesara type is not
        in the current ParamsType.

        This method is intended to be used to retrieve a field name
        when we know that current ParamsType contains the given
        Aesara type only once.

        """
        return self.fields[self.types.index(aesara_type)]

    def get_enum(self, key):
        """
        Look for a constant named ``key`` in the Aesara enumeration types
        wrapped into current ParamsType. Return value of the constant found,
        or raise an exception if either the constant is not found or
        current wrapper does not contain any Aesara enumeration type.

        **Example**::

            from aesara.graph.params_type import ParamsType
            from aesara.graph.type import EnumType, EnumList
            from aesara.scalar import Scalar

            wrapper = ParamsType(scalar=Scalar('int32'),
                                 letters=EnumType(A=1, B=2, C=3),
                                 digits=EnumList('ZERO', 'ONE', 'TWO'))
            print(wrapper.get_enum('C'))  # 3
            print(wrapper.get_enum('TWO'))  # 2

            # You can also directly do:
            print(wrapper.C)
            print(wrapper.TWO)

        """
        return self.__const_to_enum[key][key]

    def enum_from_alias(self, alias):
        """
        Look for a constant that has alias ``alias`` in the Aesara enumeration types
        wrapped into current ParamsType. Return value of the constant found,
        or raise an exception if either

        1. there is no constant with this alias,
        2. there is no constant which name is this alias, or
        3. current wrapper does not contain any Aesara enumeration type.

        **Example**::

            from aesara.graph.params_type import ParamsType
            from aesara.graph.type import EnumType, EnumList
            from aesara.scalar import Scalar

            wrapper = ParamsType(scalar=Scalar('int32'),
                                 letters=EnumType(A=(1, 'alpha'), B=(2, 'beta'), C=3),
                                 digits=EnumList(('ZERO', 'nothing'), ('ONE', 'unit'), ('TWO', 'couple')))
            print(wrapper.get_enum('C'))  # 3
            print(wrapper.get_enum('TWO'))  # 2
            print(wrapper.enum_from_alias('alpha')) # 1
            print(wrapper.enum_from_alias('nothing')) # 0

            # For the following, alias 'C' is not defined, so the method looks for
            # a constant named 'C', and finds it.
            print(wrapper.enum_from_alias('C')) # 3

        .. note::

            Unlike with constant names, you can **NOT** access constants values directly with aliases through
            ParamsType (ie. you can't write ``wrapper.alpha``). You **must** use ``wrapper.enum_from_alias()``
            method to do that.

        """
        return (
            self.__alias_to_enum[alias].fromalias(alias)
            if alias in self.__alias_to_enum
            else self.__const_to_enum[alias][alias]
        )

    def get_params(self, *objects, **kwargs):
        """
        Convenient method to extract fields values from a list of Python objects and key-value args,
        and wrap them into a :class:`Params` object compatible with current ParamsType.

        For each field defined in the current ParamsType, a value for this field
        is looked for in the given objects attributes (looking for attributes with this field name)
        and key-values args (looking for a key equal to this field name), from left to right
        (first object, then, ..., then last object, then key-value args), replacing a previous
        field value found with any value found in next step, so that only the last field value
        found is retained.

        Fields values given in objects and kwargs must be compatible with types
        associated to corresponding fields in current ParamsType.

        **Example**::

            import numpy
            from aesara.graph.params_type import ParamsType
            from aesara.tensor.type import dmatrix
            from aesara.scalar import Scalar

            class MyObject:
                def __init__(self):
                    self.a = 10
                    self.b = numpy.asarray([[1, 2, 3], [4, 5, 6]])

            params_type = ParamsType(a=Scalar('int32'), b=dmatrix, c=Scalar('bool'))

            o = MyObject()
            value_for_c = False

            # Value for c can't be retrieved from o, so we add a value for that field in kwargs.
            params = params_type.get_params(o, c=value_for_c)
            # params.a contains 10
            # params.b contains [[1, 2, 3], [4, 5, 6]]
            # params.c contains value_for_c
            print(params)

        """
        fields_values = dict()
        # We collect fields values from given objects.
        # If a field is present in many objects, only the field in the last object will be retained.
        for obj in objects:
            for field in self.fields:
                try:
                    fields_values[field] = getattr(obj, field)
                except Exception:
                    pass
        # We then collect fields values from given kwargs.
        # A field value in kwargs will replace any previous value collected from objects for this field.
        for field in self.fields:
            if field in kwargs:
                fields_values[field] = kwargs[field]
        # Then we filter the fields values and we create the Params object.
        filtered = {
            self.fields[i]: self.types[i].filter(
                fields_values[self.fields[i]], strict=False, allow_downcast=True
            )
            for i in range(self.length)
        }
        return Params(self, **filtered)

    def extended(self, **kwargs):
        """
        Return a copy of current ParamsType
        extended with attributes given in kwargs.
        New attributes must follow same rules as in
        ParamsType constructor.

        """
        self_to_dict = {self.fields[i]: self.types[i] for i in range(self.length)}
        self_to_dict.update(kwargs)
        return ParamsType(**self_to_dict)

    # Returns a Params object with expected attributes or (in strict mode) checks that data has expected attributes.
    def filter(self, data, strict=False, allow_downcast=None):
        if strict and not isinstance(data, Params):
            raise TypeError(
                f"{self}: strict mode: data should be an instance of Params."
            )
        # Filter data attributes to check if they respect the ParamsType's contract.
        filtered = {
            self.fields[i]: self.types[i].filter(
                getattr(data, self.fields[i]), strict, allow_downcast
            )
            for i in range(self.length)
        }
        return (
            data if (strict or isinstance(data, Params)) else Params(self, **filtered)
        )

    def values_eq(self, a, b):
        return all(
            self.types[i].values_eq(
                getattr(a, self.fields[i]), getattr(b, self.fields[i])
            )
            for i in range(self.length)
        )

    def values_eq_approx(self, a, b):
        return all(
            self.types[i].values_eq_approx(
                getattr(a, self.fields[i]), getattr(b, self.fields[i])
            )
            for i in range(self.length)
        )

    def c_compile_args(self, **kwargs):
        c_compile_args_list = []
        for _type in self.types:
            c_compile_args_list.extend(_type.c_compile_args(**kwargs))
        return c_compile_args_list

    def c_no_compile_args(self, **kwargs):
        c_no_compile_args_list = []
        for _type in self.types:
            c_no_compile_args_list.extend(_type.c_no_compile_args(**kwargs))
        return c_no_compile_args_list

    def c_headers(self, **kwargs):
        c_headers_list = []
        for _type in self.types:
            c_headers_list.extend(_type.c_headers(**kwargs))
        return c_headers_list

    def c_libraries(self, **kwargs):
        c_libraries_list = []
        for _type in self.types:
            c_libraries_list.extend(_type.c_libraries(**kwargs))
        return c_libraries_list

    def c_header_dirs(self, **kwargs):
        c_header_dirs_list = []
        for _type in self.types:
            c_header_dirs_list.extend(_type.c_header_dirs(**kwargs))
        return c_header_dirs_list

    def c_lib_dirs(self, **kwargs):
        c_lib_dirs_list = []
        for _type in self.types:
            c_lib_dirs_list.extend(_type.c_lib_dirs(**kwargs))
        return c_lib_dirs_list

    def c_init_code(self, **kwargs):
        c_init_code_list = []
        for _type in self.types:
            c_init_code_list.extend(_type.c_init_code(**kwargs))
        return c_init_code_list

    def c_support_code(self, **kwargs):
        sub = {"fail": "{this->setErrorOccurred(); return;}"}
        struct_name = self.name
        struct_name_defined = struct_name.upper()
        c_support_code_set = set()
        c_declare_list = []
        c_init_list = []
        c_cleanup_list = []
        c_extract_list = []
        for attribute_name, type_instance in zip(self.fields, self.types):

            try:
                # c_support_code() may return a code string or a list of code strings.
                support_code = type_instance.c_support_code()
                if not isinstance(support_code, list):
                    support_code = [support_code]
                c_support_code_set.update(support_code)
            except MethodNotDefined:
                pass

            c_declare_list.append(type_instance.c_declare(attribute_name, sub))

            c_init_list.append(type_instance.c_init(attribute_name, sub))

            c_cleanup_list.append(type_instance.c_cleanup(attribute_name, sub))

            c_extract_list.append(
                """
            void extract_%(attribute_name)s(PyObject* py_%(attribute_name)s) {
                %(extract_code)s
            }
            """
                % {
                    "attribute_name": attribute_name,
                    "extract_code": type_instance.c_extract(attribute_name, sub),
                }
            )

        struct_declare = "\n".join(c_declare_list)
        struct_init = "\n".join(c_init_list)
        struct_cleanup = "\n".join(c_cleanup_list)
        struct_extract = "\n\n".join(c_extract_list)
        struct_extract_method = """
        void extract(PyObject* object, int field_pos) {
            switch(field_pos) {
                // Extraction cases.
                %s
                // Default case.
                default:
                    PyErr_Format(PyExc_TypeError, "ParamsType: no extraction defined for a field %%d.", field_pos);
                    this->setErrorOccurred();
                    break;
            }
        }
        """ % (
            "\n".join(
                [
                    ("case %d: extract_%s(object); break;" % (i, self.fields[i]))
                    for i in range(self.length)
                ]
            )
        )
        final_struct_code = """
        /** ParamsType %(struct_name)s **/
        #ifndef %(struct_name_defined)s
        #define %(struct_name_defined)s
        struct %(struct_name)s {
            /* Attributes, */
            int %(struct_name)s_error;
            %(struct_declare)s

            /* Constructor. */
            %(struct_name)s() {
                %(struct_name)s_error = 0;
                %(struct_init)s
            }

            /* Destructor. */
            ~%(struct_name)s() {
                // cleanup() is defined below.
                cleanup();
            }

            /* Cleanup method. */
            void cleanup() {
                %(struct_cleanup)s
            }

            /* Extraction methods. */
            %(struct_extract)s

            /* Extract method. */
            %(struct_extract_method)s

            /* Other methods. */
            void setErrorOccurred() {
                ++%(struct_name)s_error;
            }
            int errorOccurred() {
                return %(struct_name)s_error;
            }
        };
        #endif
        /** End ParamsType %(struct_name)s **/
        """ % dict(
            struct_name_defined=struct_name_defined,
            struct_name=struct_name,
            struct_declare=struct_declare,
            struct_init=struct_init,
            struct_cleanup=struct_cleanup,
            struct_extract=struct_extract,
            struct_extract_method=struct_extract_method,
        )

        return list(sorted(list(c_support_code_set))) + [final_struct_code]

    def c_code_cache_version(self):
        return ((3,), tuple(t.c_code_cache_version() for t in self.types))

    # As this struct has constructor and destructor, it could be instantiated
    # on stack, but current implementations of C ops will then pass the
    # instance by value to functions, so it's better to work directly with
    # pointers.

    def c_declare(self, name, sub, check_input=True):
        return """
        %(struct_name)s* %(name)s;
        """ % dict(
            struct_name=self.name, name=name
        )

    def c_init(self, name, sub):
        # NB: It seems c_init() is not called for an op param.
        # So the real initialization is done at top of c_extract.
        return f"""
        {name} = NULL;
        """

    def c_cleanup(self, name, sub):
        return f"""
        delete {name};
        {name} = NULL;
        """

    def c_extract(self, name, sub, check_input=True, **kwargs):
        return """
        /* Seems c_init() is not called for a op param. So I call `new` here. */
        %(name)s = new %(struct_name)s;

        { // This need a separate namespace for Clinker
        const char* fields[] = {%(fields_list)s};
        if (py_%(name)s == Py_None) {
            PyErr_SetString(PyExc_ValueError, "ParamsType: expected an object, not None.");
            %(fail)s
        }
        for (int i = 0; i < %(length)s; ++i) {
            PyObject* o = PyDict_GetItemString(py_%(name)s, fields[i]);
            if (o == NULL) {
                PyErr_Format(PyExc_TypeError, "ParamsType: missing expected attribute \\"%%s\\" in object.", fields[i]);
                %(fail)s
            }
            %(name)s->extract(o, i);
            if (%(name)s->errorOccurred()) {
                /* The extract code from attribute type should have already raised a Python exception,
                 * so we just print the attribute name in stderr. */
                fprintf(stderr, "\\nParamsType: error when extracting value for attribute \\"%%s\\".\\n", fields[i]);
                %(fail)s
            }
        }
        }
        """ % dict(
            name=name,
            struct_name=self.name,
            length=self.length,
            fail=sub["fail"],
            fields_list='"%s"' % '", "'.join(self.fields),
        )

    def c_sync(self, name, sub):
        # FIXME: Looks like we need to decrement a reference count our two.
        # Not sure if this means we should not consider this a `CType`.
        # It seems like this means this actually means that objects of this
        # `Type` cannot be (compiled) graph _outputs_, because that's when
        # `CType.c_sync` is used.
        raise NotImplementedError("Variables of this type cannot be graph outputs")

max_stars_repo_name: fshart/aesara

max_issues_repo_name: fshart/aesara

max_forks_repo_name: fshart/aesara


text:
import torch
import torch.nn as nn
import torch.optim as optim
import torchvision.transforms as transforms
import os
from torch.autograd import Variable
import argparse
import numpy as np
from torch.optim.lr_scheduler import *

from model.resnet import resnet101
from data_pre.FashionAI import fashion

parser=argparse.ArgumentParser()
parser.add_argument('--workers',type=int,default=2)
parser.add_argument('--batchSize',type=int,default=64)
parser.add_argument('--nepoch',type=int,default=11)
parser.add_argument('--lr',type=float,default=0.001)
parser.add_argument('--gpu',type=str,default='7')
parser.add_argument('--attr',type=str,default='collar_design_labels')
opt=parser.parse_args()
print(opt)
os.environ["CUDA_VISIBLE_DEVICES"]=opt.gpu

transform_train=transforms.Compose([
	transforms.Resize((256,256)),
	transforms.RandomCrop((224,224)),
	transforms.RandomHorizontalFlip(),
	transforms.ToTensor(),
	transforms.Normalize((0.485,0.456,0.406),(0.229,0.224,0.225))
])

transform_val=transforms.Compose([
	transforms.Resize((224,224)),
	transforms.ToTensor(),
	transforms.Normalize((0.485,0.456,0.406),(0.229,0.224,0.225))
])

trainset=fashion('/home/yhf/Challenge/FashionAI/STL_FashionAI/data/2base/Annotations/sum_labels.csv',transform_train,opt.attr,train=True)
trainloader=torch.utils.data.DataLoader(trainset,batch_size=opt.batchSize,shuffle=True,num_workers=opt.workers)
valset=fashion('/home/yhf/Challenge/FashionAI/STL_FashionAI/data/2base/Annotations/sum_labels.csv',transform_val,opt.attr,train=False)
valloader=torch.utils.data.DataLoader(valset,batch_size=opt.batchSize,shuffle=False,num_workers=opt.workers)
AttrNum={
	"coat_length_labels":8,
	"collar_design_labels":5,
	"lapel_design_labels":5,
	"neck_design_labels":5,
	"neckline_design_labels":10,
	"pant_length_labels":6,
	"skirt_length_labels":6,
	"sleeve_length_labels":9
}

model=resnet101(pretrained=True)
model.fc=nn.Linear(2048,AttrNum[opt.attr])
model.cuda()
optimizer=torch.optim.SGD(model.parameters(),lr=opt.lr,momentum=0.9,weight_decay=5e-4)
scheduler=StepLR(optimizer,step_size=3)
criterion=nn.CrossEntropyLoss()
criterion.cuda()

def train(epoch):
	print('\nTrain Epoch:%d' % epoch)
	scheduler.step()
	model.train()
	for batch_idx, (img,label) in enumerate(trainloader):
		image=Variable(img.cuda())
		label=Variable(label.cuda())
		optimizer.zero_grad()
		out=model(image)
		loss=criterion(out,label)
		loss.backward()
		optimizer.step()
		if batch_idx%20==0:
			print("Epoch: %d [%d:%d] loss: %f" % (epoch,batch_idx,len(trainloader),loss.mean()))

def val(epoch):
	print('\nTest Epoch:%d'%epoch)
	model.eval()
	total=0
	correct=0
	for batch_idx, (img,label) in enumerate(valloader):
		image=Variable(img.cuda(),volatile=True)
		label=Variable(label.cuda())
		out=model(image)
		_,predict=torch.max(out.data,1)
		total+=image.size(0)
		correct+=predict.eq(label.data).cpu().sum()
	print("Acc:%f" % ((1.0*correct)/total))

for epoch in range(opt.nepoch):
	train(epoch)
	val(epoch)
torch.save(model.state_dict(),'ckp/model_task_%s.pth' % opt.attr)

max_stars_repo_name: JackwithWilshere/FashionAI-

max_issues_repo_name: espectre/FashionAI

max_forks_repo_name: espectre/FashionAI


text:
# -*- coding: utf-8 -*-
"""
This module takes a trained voting classifier and trains it.
Usually, this will be used to train on a larger dataset, for which not the
expensive hyperopt search should be performed.

This assumed that we read a VotingClassifier, which has the private _fit and _calibrate
methods.
"""
import os
import time

import click
import numpy as np
from comet_ml import Experiment
from joblib import dump, load
from sklearn.metrics import (
    accuracy_score,
    balanced_accuracy_score,
    f1_score,
    precision_score,
    recall_score,
)
from sklearn.preprocessing import MinMaxScaler, RobustScaler, StandardScaler

class TrainVotingClassifier:
    def __init__(  # pylint:disable=too-many-arguments
        self,
        votingclassifier,
        X,
        y,
        calibration=None,
        voting=None,
        outdir=None,
        scaler="minmax",
        validX=None,
        validy=None,
    ):
        self.votingclassifier = votingclassifier
        self.X = X
        self.y = y
        self.validX = validX
        self.validy = validy
        self.y = self.y.astype(np.int)
        self.validy = self.validy.astype(np.int)
        # print(
        #     f'the class labels in the validation data are {np.unique(self.validy)}, the ones in the training set are {np.unique(self.y)}'
        # )
        assert len(np.unique(self.validy)) <= len(np.unique(self.y))

        assert len(self.X) == len(self.y)

        self.calibration = calibration
        self.voting = voting
        self.votingclassifier.voting = self.voting
        self.outdir = outdir
        if self.outdir is None:
            self.outdir = os.getcwd()
        self.traintime = None

        if scaler == "robust":
            self.scalername = "robust"
            self.scaler = RobustScaler()
        elif scaler == "standard":
            self.scalername = "standard"
            self.scaler = StandardScaler()
        elif scaler == "minmax":
            self.scalername = "minmax"
            self.scaler = MinMaxScaler()

        # initialize voting and calibration if None based on the settings of the classifier
        if self.calibration is None:
            self.calibration = self.votingclassifier.voting

        if self.voting is None:
            self.voting = self.votingclassifier.voting

        self.experiment = Experiment(
            api_key=os.getenv("COMET_API_KEY", None),
            project_name="mof-oxidation-states",
        )

        # directly log some parameters of the experiment

        self.experiment.log_parameter("voting", self.voting)
        self.experiment.log_parameter("calibration", self.calibration)
        self.experiment.log_parameter("scaler", self.scaler)
        self.experiment.log_parameter("training_size", len(self.y))

    @classmethod
    def from_files(  # pylint:disable=too-many-arguments, too-many-locals
        cls,
        joblibpath,
        featurematrixpath,
        labelpath,
        calibration=None,
        voting=None,
        outdir=None,
        scaler="standard",
        validX=None,
        validy=None,
    ):
        model = load(joblibpath)
        X = np.load(featurematrixpath)
        y = np.load(labelpath)
        validXdata = np.load(validX)
        validydata = np.load(validy)

        # print(
        #     f'the class labels in the validation data are {np.unique(validy)}, the ones in the training set are {np.unique(y)}'
        # )
        return cls(
            model, X, y, calibration, voting, outdir, scaler, validXdata, validydata
        )

    def _fit(self):
        self.X = self.scaler.fit_transform(self.X)
        self.validX = self.scaler.transform(self.validX)

        startime = time.process_time()
        self.votingclassifier._fit(self.X, self.y)  # pylint:disable=protected-access
        # use validation set to do probability calibration

        # print(f'sent {np.unique(self.validy)} to calibration')
        self.votingclassifier._calibrate_base_estimators(  # pylint:disable=protected-access
            self.calibration, self.validX, self.validy
        )
        endtime = time.process_time()
        self.traintime = endtime - startime
        self.experiment.log_metric("training_time", self.traintime)

    def _train_metrics(self):
        predict = self.votingclassifier.predict(self.X)

        accuracy = accuracy_score(self.y, predict)
        f1_micro = f1_score(self.y, predict, average="micro")
        f1_macro = f1_score(self.y, predict, average="macro")
        balanced_accuracy = balanced_accuracy_score(self.y, predict)
        precision = precision_score(self.y, predict, average="micro")
        recall = recall_score(self.y, predict, average="micro")

        self.experiment.log_metric("accuracy", accuracy)
        self.experiment.log_metric("f1_micro", f1_micro)
        self.experiment.log_metric("f1_macro", f1_macro)
        self.experiment.log_metric("balanced_accuracy", balanced_accuracy)
        self.experiment.log_metric("precision", precision)
        self.experiment.log_metric("recall", recall)

    def _dump(self):
        if not os.path.exists(self.outdir):
            os.mkdir(self.outdir)

        self.votingclassifier._check_is_fitted()  # pylint:disable=protected-access
        dump(
            self.votingclassifier, os.path.join(self.outdir, "votingclassifier.joblib")
        )
        dump(self.scaler, os.path.join(self.outdir, "scaler.joblib"))

    def return_models(self):
        return self.votingclassifier, self.scaler

    def train(self):
        self._fit()
        self._train_metrics()
        self._dump()

@click.command("cli")
@click.argument("modelpath")
@click.argument("featurepath")
@click.argument("labelpath")
@click.argument("calibration")
@click.argument("voting")
@click.argument("outdir")
@click.argument("scaler")
@click.argument("validx")
@click.argument("validy")
def main(  # pylint:disable=too-many-arguments
    modelpath,
    featurepath,
    labelpath,
    calibration,
    voting,
    outdir,
    scaler,
    validx,
    validy,
):

    vc = TrainVotingClassifier.from_files(
        modelpath,
        featurepath,
        labelpath,
        calibration,
        voting,
        outdir,
        scaler,
        validx,
        validy,
    )
    vc.train()

if __name__ == "__main__":
    main()  # pylint:disable=no-value-for-parameter


text:
import numpy as np
import cv2

camera = cv2.VideoCapture(0)
out = cv2.VideoWriter('capturdVid.mkv', cv2.VideoWriter_fourcc(*'DIVX'), 50, (640, 480))
while True:
	ret, frame = camera.read()
	cv2.imshow('video', frame)
	out.write(frame)
	if cv2.waitKey(1) == 'a':
		break
camera.release()
out.release()
cv2.destroyAllWindows()


text:
#!/usr/bin/env python

# Python libs
import numpy
import tf

# ROS libraries
import roslib
import rospy

# ROS messages
from std_msgs.msg import Int16
from std_msgs.msg import Float32
from std_msgs.msg import Float64

# Class that does all the work
class Target_Motor:

	# Initialize ROS publishers, subscribers and variables
	def __init__(self):

		# Topics where we publish
		self.Move_A_Publisher= rospy.Publisher('/Arm_Motor/vel_cmd', Float64, queue_size=1)

		# Subscribed topics
		self.Encoder_A_Subscriber = rospy.Subscriber('/Arm_Motor/target_cmd', Int16, self.Encoder_A_Processor)

		# Variables
		self.Motor_Vel=[0.5]
		self.Encoder_Data = [0]

		# Gives time to publishers and subscribers to init
		rospy.sleep(3)

	# Callback function for a new msg in the subscribed topic
	def Encoder_A_Processor(self,Target_Data):

		# Store actual encoder value
		self.Encoder_Data[0]=rospy.wait_for_message('Encoder_Arm_Sensor', Int16)

		# Declare desired encoder value
		# This supposse relative movemt of the arm
		# Usefull if you want to specify a joint movement by number of rotations
		Target_Angle= self.Encoder_Data[0].data+Target_Data.data

		# Untill the encoder value is equal to the desire angle
		while (self.Encoder_Data[0].data!=Target_Angle):

			# Move up if it is down
			if self.Encoder_Data[0].data>Target_Angle:

				# Publish in /Arm_Motor/vel_cmd topic a msg equal to float -0.5
				self.Move_A_Publisher.publish(-self.Motor_Vel[0])

			# Move down if it up
			elif self.Encoder_Data[0].data<Target_Angle:

				# Publish in /Arm_Motor/vel_cmd topic a msg equal to float 0.5
				self.Move_A_Publisher.publish(self.Motor_Vel[0])

			# Store new encoder value
			self.Encoder_Data[0]=rospy.wait_for_message('Encoder_Arm_Sensor', Int16)

		# When equal, dont move it more
		self.Move_A_Publisher.publish(0.0)

# Main
def main():

		# Initializes and cleanup ROS node
    rospy.init_node('Target_Motor', anonymous=True)
    TM = Target_Motor()

    try:
    		# Listens for new msgs in the subscribed topics
        rospy.spin()
    except KeyboardInterrupt:
        print "Shutting down Target Motor"

if __name__ == '__main__':
    main()

max_stars_repo_name: imstevenpm/lmev

max_issues_repo_name: imstevenpm/lmev

max_forks_repo_name: imstevenpm/lmev