import copy
from collections import defaultdict
import random
import netaddr

from datadiff import diff
import networkx

from faucet import faucet_pipeline
from faucet import valve_of
from faucet.conf import Conf, test_config_condition
from faucet.valve import SUPPORTED_HARDWARE
from faucet.faucet_pipeline import ValveTableConfig
from faucet.valve_table import ValveTable, ValveGroupTable
from faucet import valve_packet

class NullRyuDatapath:
    """Placeholder Ryu Datapath."""

    ofproto = valve_of.ofp

# Documentation generated using documentation_generator.py
# For attributues to be included in documentation they must
# have a default value, and their descriptor must come
# immediately after being set. See below for example.
class DP(Conf):
    """Stores state related to a datapath controlled by Faucet, including
configuration.
"""

    mutable_attrs = frozenset(['stack', 'vlans'])

    # Values that are set to None will be set using set_defaults
    # they are included here for testing and informational purposes
    defaults = {
        'dp_id': None,
        # Name for this dp, used for stats reporting and configuration
        'name': None,
        'interfaces': {},
        'interface_ranges': {},
        # How much to offset default priority by
        'priority_offset': 0,
        # Some priority values
        'lowest_priority': None,
        'low_priority': None,
        'high_priority': None,
        'highest_priority': None,
        'cookie': 1524372928,
        # Identification cookie value to allow for multiple controllers to control the same datapath
        'timeout': 300,
        # inactive MAC timeout
        'description': None,
        # description, strictly informational
        'hardware': 'Open vSwitch',
        # The hardware maker (for chosing an openflow driver)
        'arp_neighbor_timeout': 250,
        # ARP and neighbor timeout (seconds)
        'ofchannel_log': None,
        # OF channel log
        'stack': None,
        # stacking config, when cross connecting multiple DPs
        'ignore_learn_ins': 10,
        # Ignore every approx nth packet for learning.
        # 2 will ignore 1 out of 2 packets; 3 will ignore 1 out of 3 packets.
        # This limits control plane activity when learning new hosts rapidly.
        # Flooding will still be done by the dataplane even with a packet
        # is ignored for learning purposes.
        'drop_broadcast_source_address': True,
        # By default drop packets with a broadcast source address
        'drop_spoofed_faucet_mac': True,
        # By default drop packets on datapath spoofing the FAUCET_MAC
        'group_table': False,
        # Use GROUP tables for VLAN flooding
        'group_table_routing': False,
        # Use GROUP tables for routing (nexthops)
        'max_hosts_per_resolve_cycle': 5,
        # Max hosts to try to resolve per gateway resolution cycle.
        'max_host_fib_retry_count': 10,
        # Max number of times to retry resolution of a host FIB route.
        'max_resolve_backoff_time': 32,
        # Max number of seconds to back off to when resolving nexthops.
        'packetin_pps': None,
        # Ask switch to rate limit packet pps. TODO: Not supported by OVS in 2.7.0
        'learn_jitter': 10,
        # Jitter learn timeouts by up to this many seconds
        'learn_ban_timeout': 10,
        # When banning/limiting learning, wait this many seconds before learning can be retried
        'advertise_interval': 30,
        # How often to advertise (eg. IPv6 RAs)
        'proactive_learn_v4': True,
        # whether proactive learning is enabled for IPv4 nexthops
        'proactive_learn_v6': True,
        # whether proactive learning is enabled for IPv6 nexthops
        'use_idle_timeout': False,
        # Turn on/off the use of idle timeout for src_table, default OFF.
        'lldp_beacon': {},
        # Config for LLDP beacon service.
        'metrics_rate_limit_sec': 0,
        # Rate limit metric updates - don't update metrics if last update was less than this many seconds ago.
        'faucet_dp_mac': valve_packet.FAUCET_MAC,
        # MAC address of packets sent by FAUCET, not associated with any VLAN.
        'combinatorial_port_flood': False,
        # if True, use a seperate output flow for each input port on this VLAN.
        'lacp_timeout': 30,
        # Number of seconds without a LACP message when we consider a LACP group down.
        'dp_acls': None,
        # List of dataplane ACLs (overriding per port ACLs).
        'dot1x': None,
        # Experimental dot1x configuration.
        'table_sizes': {},
        # Table sizes for TFM switches.
        'min_wildcard_table_size': 32,
        # Minimum table size for wildcard tables.
        'max_wildcard_table_size': 1024 + 256,
        # Maximum table size for wildcard tables.
        'global_vlan': 0,
        # Reserved VID for internal global router VLAN.
        }

    defaults_types = {
        'dp_id': int,
        'name': str,
        'interfaces': dict,
        'interface_ranges': dict,
        'priority_offset': int,
        'lowest_priority': int,
        'low_priority': int,
        'high_priority': int,
        'highest_priority': int,
        'cookie': int,
        'timeout': int,
        'description': str,
        'hardware': str,
        'arp_neighbor_timeout': int,
        'ofchannel_log': str,
        'stack': dict,
        'ignore_learn_ins': int,
        'drop_broadcast_source_address': bool,
        'drop_spoofed_faucet_mac': bool,
        'drop_bpdu': bool,
        'drop_lldp': bool,
        'group_table': bool,
        'group_table_routing': bool,
        'max_hosts_per_resolve_cycle': int,
        'max_host_fib_retry_count': int,
        'max_resolve_backoff_time': int,
        'packetin_pps': int,
        'learn_jitter': int,
        'learn_ban_timeout': int,
        'advertise_interval': int,
        'proactive_learn_v4': bool,
        'proactive_learn_v6': bool,
        'use_idle_timeout': bool,
        'lldp_beacon': dict,
        'metrics_rate_limit_sec': int,
        'faucet_dp_mac': str,
        'combinatorial_port_flood': bool,
        'dp_acls': list,
        'dot1x': dict,
        'table_sizes': dict,
        'min_wildcard_table_size': int,
        'max_wildcard_table_size': int,
        'global_vlan': int,
    }

    default_table_sizes_types = {
        'port_acl': int,
        'vlan': int,
        'vlan_acl': int,
        'eth_src': int,
        'ipv4_fib': int,
        'ipv6_fib': int,
        'vip': int,
        'eth_dst': int,
        'flood': int,
        'egress_acl': int,
    }

    stack_defaults_types = {
        'priority': int,
    }

    lldp_beacon_defaults_types = {
        'send_interval': int,
        'max_per_interval': int,
        'system_name': str,
    }

    dot1x_defaults_types = {
        'nfv_intf': str,
    }

    wildcard_table = ValveTable(
        valve_of.ofp.OFPTT_ALL, 'all', ValveTableConfig('all'), flow_cookie=0)

    def __init__(self, _id, dp_id, conf):
        """Constructs a new DP object"""
        self.acls = None
        self.acls_in = None
        self.acls_out = None
        self.advertise_interval = None
        self.arp_neighbor_timeout = None
        self.bgp_local_address = None
        self.bgp_neighbor_as = None
        self.bgp_routerid = None
        self.combinatorial_port_flood = None
        self.configured = False
        self.cookie = None
        self.description = None
        self.dot1x = None
        self.dp_acls = None
        self.dp_id = None
        self.drop_broadcast_source_address = None
        self.drop_spoofed_faucet_mac = None
        self.dyn_last_coldstart_time = None
        self.dyn_running = False
        self.dyn_up_ports = None
        self.faucet_dp_mac = None
        self.global_vlan = None
        self.groups = None
        self.group_table = False
        self.group_table_routing = False
        self.hardware = None
        self.high_priority = None
        self.highest_priority = None
        self.ignore_learn_ins = None
        self.interface_ranges = None
        self.interfaces = None
        self.lacp_timeout = None
        self.learn_ban_timeout = None
        self.learn_jitter = None
        self.lldp_beacon = None
        self.low_priority = None
        self.lowest_priority = None
        self.max_host_fib_retry_count = None
        self.max_hosts_per_resolve_cycle = None
        self.max_resolve_backoff_time = None
        self.meters = None
        self.metrics_rate_limit_sec = None
        self.name = None
        self.ofchannel_log = None
        self.output_only_ports = None
        self.packetin_pps = None
        self.ports = None
        self.priority_offset = None
        self.proactive_learn_v4 = None
        self.proactive_learn_v6 = None
        self.proactive_nd_limit = None
        self.routers = None
        self.stack = None
        self.stack_ports = None
        self.tables = None
        self.timeout = None
        self.unicast_flood = None
        self.use_idle_timeout = None
        self.vlans = None
        self.min_wildcard_table_size = None
        self.max_wildcard_table_size = None

        self.acls = {}
        self.vlans = {}
        self.ports = {}
        self.routers = {}
        self.stack_ports = []
        self.output_only_ports = []
        self.lldp_beacon_ports = []
        self.tables = {}
        self.meters = {}
        self.lldp_beacon = {}
        self.table_sizes = {}
        self.dyn_up_ports = set()
        super(DP, self).__init__(_id, dp_id, conf)

    def __str__(self):
        return self.name

    def check_config(self):
        super(DP, self).check_config()
        test_config_condition(not isinstance(self.dp_id, int), (
            'dp_id must be %s not %s' % (int, type(self.dp_id))))
        test_config_condition(self.dp_id < 0 or self.dp_id > 2**64-1, (
            'DP ID %s not in valid range' % self.dp_id))
        test_config_condition(not netaddr.valid_mac(self.faucet_dp_mac), (
            'invalid MAC address %s' % self.faucet_dp_mac))
        test_config_condition(self.group_table and self.group_table_routing, (
            'groups for routing and other functions simultaneously not supported'))
        test_config_condition(not (self.interfaces or self.interface_ranges), (
            'DP %s must have at least one interface' % self))
        # To prevent L2 learning from timing out before L3 can refresh
        test_config_condition(self.timeout < self.arp_neighbor_timeout, (
            'L2 timeout must be >= L3 timeout'))
        if self.lldp_beacon:
            self._check_conf_types(self.lldp_beacon, self.lldp_beacon_defaults_types)
            test_config_condition('send_interval' not in self.lldp_beacon, (
                'lldp_beacon send_interval not set'))
            test_config_condition('max_per_interval' not in self.lldp_beacon, (
                'lldp_beacon max_per_interval not set'))
            self.lldp_beacon = self._set_unknown_conf(
                self.lldp_beacon, self.lldp_beacon_defaults_types)
            if self.lldp_beacon['system_name'] is None:
                self.lldp_beacon['system_name'] = self.name
        if self.stack:
            self._check_conf_types(self.stack, self.stack_defaults_types)
        if self.dot1x:
            self._check_conf_types(self.dot1x, self.dot1x_defaults_types)
        self._check_conf_types(self.table_sizes, self.default_table_sizes_types)

    def _configure_tables(self, override_table_config, valve_cl, vlan_port_factor):
        """Configure FAUCET pipeline with tables."""
        tables = {}
        self.groups = ValveGroupTable()
        relative_table_id = 0
        for table_id, table_config in enumerate(faucet_pipeline.FAUCET_PIPELINE):
            table_name = table_config.name
            if table_name in override_table_config:
                table_config = override_table_config[table_name]
            size = self.table_sizes.get(table_name, self.min_wildcard_table_size)
            if table_config.vlan_port_scale:
                size = max(size, int(vlan_port_factor * float(table_config.vlan_port_scale)))
            if not table_config.exact_match:
                size = min(size, self.max_wildcard_table_size)
                size = int(size / self.min_wildcard_table_size) * self.min_wildcard_table_size
            table_config.size = size
            if table_config.match_types:
                if not valve_cl.STATIC_TABLE_IDS:
                    table_id = relative_table_id
                tables[table_name] = ValveTable(
                    table_id, table_name, table_config, self.cookie,
                    notify_flow_removed=self.use_idle_timeout)
                relative_table_id += 1
        self.tables = tables

    def set_defaults(self):
        super(DP, self).set_defaults()
        self._set_default('dp_id', self._id)
        self._set_default('name', str(self._id))
        self._set_default('lowest_priority', self.priority_offset)
        self._set_default('low_priority', self.priority_offset + 9000)
        self._set_default('high_priority', self.low_priority + 1)
        self._set_default('highest_priority', self.high_priority + 98)
        self._set_default('description', self.name)

    def table_by_id(self, table_id):
        tables = [table for table in list(self.tables.values()) if table_id == table.table_id]
        if tables:
            return tables[0]
        return None

    def match_tables(self, match_type):
        """Return list of tables with matches of a specific match type."""
        match_tables = []
        for table in list(self.tables.values()):
            if table.match_types is not None:
                if match_type in table.match_types:
                    match_tables.append(table)
            else:
                match_tables.append(table)
        return match_tables

    def in_port_tables(self):
        """Return list of tables that specify in_port as a match."""
        return self.match_tables('in_port')

    def add_acl(self, acl_ident, acl):
        """Add an ACL to this DP."""
        self.acls[acl_ident] = acl

    def add_router(self, router_ident, router):
        """Add a router to this DP."""
        self.routers[router_ident] = router

    def add_port(self, port):
        """Add a port to this DP."""
        port_num = port.number
        self.ports[port_num] = port
        if port.output_only:
            self.output_only_ports.append(port)
        elif port.stack is not None:
            self.stack_ports.append(port)
        if port.lldp_beacon_enabled():
            self.lldp_beacon_ports.append(port)

    def lldp_beacon_send_ports(self, now):
        """Return list of ports to send LLDP packets; stacked ports always send LLDP."""
        send_ports = []
        if self.lldp_beacon:
            priority_ports = {
                port for port in self.stack_ports
                if port.running() and port.lldp_beacon_enabled()}
            cutoff_beacon_time = now - self.lldp_beacon['send_interval']
            nonpriority_ports = {
                port for port in self.lldp_beacon_ports
                if port.running() and (
                    port.dyn_last_lldp_beacon_time is None or
                    port.dyn_last_lldp_beacon_time < cutoff_beacon_time)}
            nonpriority_ports -= priority_ports
            send_ports.extend(list(priority_ports))
            send_ports.extend(list(nonpriority_ports)[:self.lldp_beacon['max_per_interval']])
            random.shuffle(send_ports)
        return send_ports

    @staticmethod
    def modify_stack_topology(graph, dp, port, add=True):
        """Add/remove an edge to the stack graph which originates from this dp and port."""

        def canonical_edge(dp, port):
            peer_dp = port.stack['dp']
            peer_port = port.stack['port']
            sort_edge_a = (
                dp.name, port.name, dp, port)
            sort_edge_z = (
                peer_dp.name, peer_port.name, peer_dp, peer_port)
            sorted_edge = sorted((sort_edge_a, sort_edge_z))
            edge_a, edge_b = sorted_edge[0][2:], sorted_edge[1][2:]
            return edge_a, edge_b

        def make_edge_name(edge_a, edge_z):
            edge_a_dp, edge_a_port = edge_a
            edge_z_dp, edge_z_port = edge_z
            return '%s:%s-%s:%s' % (
                edge_a_dp.name, edge_a_port.name,
                edge_z_dp.name, edge_z_port.name)

        def make_edge_attr(edge_a, edge_z):
            edge_a_dp, edge_a_port = edge_a
            edge_z_dp, edge_z_port = edge_z
            return {
                'dp_a': edge_a_dp, 'port_a': edge_a_port,
                'dp_z': edge_z_dp, 'port_z': edge_z_port}

        edge = canonical_edge(dp, port)
        edge_a, edge_z = edge
        edge_name = make_edge_name(edge_a, edge_z)
        edge_attr = make_edge_attr(edge_a, edge_z)
        edge_a_dp, _ = edge_a
        edge_z_dp, _ = edge_z
        if add:
            graph.add_edge(
                edge_a_dp.name, edge_z_dp.name,
                key=edge_name, port_map=edge_attr)
        elif (edge_a_dp.name, edge_z_dp.name, edge_name) in graph.edges:
            graph.remove_edge(edge_a_dp.name, edge_z_dp.name, edge_name)

        return edge_name

    @classmethod
    def add_stack_link(cls, graph, dp, port):
        """Add a stack link to the stack graph."""
        return cls.modify_stack_topology(graph, dp, port)

    @classmethod
    def remove_stack_link(cls, graph, dp, port):
        """Remove a stack link to the stack graph."""
        return cls.modify_stack_topology(graph, dp, port, False)

    def resolve_stack_topology(self, dps):
        """Resolve inter-DP config for stacking."""
        root_dp = None
        stack_dps = []
        for dp in dps:
            if dp.stack is not None:
                stack_dps.append(dp)
                if 'priority' in dp.stack:
                    test_config_condition(dp.stack['priority'] <= 0, (
                        'stack priority must be > 0'))
                    test_config_condition(root_dp is not None, 'cannot have multiple stack roots')
                    root_dp = dp
                    for vlan in list(dp.vlans.values()):
                        test_config_condition(vlan.faucet_vips, (
                            'routing + stacking not supported'))

        if root_dp is None:
            test_config_condition(stack_dps, 'stacking enabled but no root_dp')
            return

        edge_count = {}

        graph = networkx.MultiGraph()
        for dp in dps:
            if dp.stack_ports:
                graph.add_node(dp.name)
                for port in dp.stack_ports:
                    edge_name = self.add_stack_link(graph, dp, port)
                    if edge_name not in edge_count:
                        edge_count[edge_name] = 0
                    edge_count[edge_name] += 1
        if graph.size():
            for edge_name, count in list(edge_count.items()):
                test_config_condition(count != 2, '%s defined only in one direction' % edge_name)
            if self.name in graph:
                if self.stack is None:
                    self.stack = {}
                self.stack['root_dp'] = root_dp
                self.stack['graph'] = graph
                longest_path_to_root_len = 0
                for dp in graph.nodes():
                    path_to_root_len = len(self.shortest_path(root_dp.name, src_dp=dp))
                    test_config_condition(
                        path_to_root_len == 0, '%s not connected to stack' % dp)
                    longest_path_to_root_len = max(
                        path_to_root_len, longest_path_to_root_len)
                self.stack['longest_path_to_root_len'] = longest_path_to_root_len

    def shortest_path(self, dest_dp, src_dp=None):
        """Return shortest path to a DP, as a list of DPs."""
        if src_dp is None:
            src_dp = self.name
        if self.stack is not None and 'root_dp' in self.stack:
            try:
                return networkx.shortest_path(
                    self.stack['graph'], src_dp, dest_dp)
            except networkx.exception.NetworkXNoPath:
                pass
        return []

    def shortest_path_to_root(self):
        """Return shortest path to root DP, as list of DPs."""
        # TODO: root_dp will be None, if stacking is enabled but the root DP is down.
        if self.stack is not None and 'root_dp' in self.stack:
            root_dp = self.stack['root_dp']
            if root_dp is not None and root_dp != self:
                return self.shortest_path(root_dp.name)
        return []

    def is_stack_root(self):
        """Return True if this DP is the root of the stack."""
        return 'priority' in self.stack

    def is_stack_edge(self):
        """Return True if this DP is a stack edge."""
        if self.stack and 'longest_path_to_root_len' in self.stack:
            return self.stack['longest_path_to_root_len'] == len(self.shortest_path_to_root())
        return False

    def peer_stack_up_ports(self, peer_dp):
        """Return list of stack ports that are up towards a peer."""
        return [port for port in self.stack_ports if port.running() and (
            port.stack['dp'].name == peer_dp)]

    def shortest_path_port(self, dest_dp):
        """Return first port on our DP, that is the shortest path towards dest DP."""
        shortest_path = self.shortest_path(dest_dp)
        if len(shortest_path) > 1:
            peer_dp = shortest_path[1]
            peer_dp_ports = self.peer_stack_up_ports(peer_dp)
            if peer_dp_ports:
                return peer_dp_ports[0]
        return None

    def reset_refs(self, vlans=None):
        if vlans is None:
            vlans = self.vlans
        self.vlans = {}
        for vlan in list(vlans.values()):
            vlan.reset_ports(list(self.ports.values()))
            if vlan.get_ports():
                self.vlans[vlan.vid] = vlan

    def resolve_port(self, port_name):
        """Resolve a port by number or name."""
        if isinstance(port_name, int):
            if port_name in self.ports:
                return self.ports[port_name]
        elif isinstance(port_name, str):
            resolved_ports = [port for port in list(self.ports.values()) if port_name == port.name]
            if resolved_ports:
                return resolved_ports[0]
        return None

    def finalize_config(self, dps):
        """Perform consistency checks after initial config parsing."""

        dp_by_name = {}
        vlan_by_name = {}

        def resolve_ports(port_names):
            """Resolve list of ports, by port by name or number."""
            resolved_ports = []
            for port_name in port_names:
                port = self.resolve_port(port_name)
                if port is not None:
                    resolved_ports.append(port)
            return resolved_ports

        def resolve_vlan(vlan_name):
            """Resolve VLAN by name or VID."""
            test_config_condition(not isinstance(vlan_name, (str, int)), (
                'VLAN must be type %s or %s not %s' % (str, int, type(vlan_name))))
            if vlan_name in vlan_by_name:
                return vlan_by_name[vlan_name]
            if vlan_name in self.vlans:
                return self.vlans[vlan_name]
            return None

        def resolve_stack_dps():
            """Resolve DP references in stacking config."""
            port_stack_dp = {}
            for port in self.stack_ports:
                stack_dp = port.stack['dp']
                test_config_condition(stack_dp not in dp_by_name, (
                    'stack DP %s not defined' % stack_dp))
                port_stack_dp[port] = dp_by_name[stack_dp]
            for port, dp in list(port_stack_dp.items()):
                port.stack['dp'] = dp
                stack_port = dp.resolve_port(port.stack['port'])
                test_config_condition(stack_port is None, (
                    'stack port %s not defined in DP %s' % (port.stack['port'], dp.name)))
                port.stack['port'] = stack_port

        def resolve_mirror_destinations():
            """Resolve mirror port references and destinations."""
            mirror_from_port = defaultdict(list)
            for mirror_port in list(self.ports.values()):
                if mirror_port.mirror is not None:
                    mirrored_ports = resolve_ports(mirror_port.mirror)
                    test_config_condition(len(mirrored_ports) != len(mirror_port.mirror), (
                        'port mirror not defined in DP %s' % self.name))
                    for mirrored_port in mirrored_ports:
                        mirror_from_port[mirrored_port].append(mirror_port)

            # TODO: confusingly, mirror at config time means what ports to mirror from.
            # But internally we use as a list of ports to mirror to.
            for mirrored_port, mirror_ports in list(mirror_from_port.items()):
                mirrored_port.mirror = []
                for mirror_port in mirror_ports:
                    mirrored_port.mirror.append(mirror_port.number)
                    mirror_port.output_only = True

        def resolve_override_output_ports():
            """Resolve override output ports."""
            for port_no, port in list(self.ports.items()):
                if port.override_output_port:
                    port.override_output_port = self.resolve_port(port.override_output_port)
                    test_config_condition(not port.override_output_port, (
                        'override_output_port port not defined'))
                    self.ports[port_no] = port

        def resolve_acl(acl_in, vid):
            """Resolve an individual ACL."""
            test_config_condition(acl_in not in self.acls, (
                'missing ACL %s in DP: %s' % (acl_in, self.name)))
            acl = self.acls[acl_in]
            def resolve_port_cb(port_name):
                port = self.resolve_port(port_name)
                if port:
                    return port.number
                return port

            acl.resolve_ports(resolve_port_cb)

            for meter_name in acl.get_meters():
                test_config_condition(meter_name not in self.meters, (
                    'meter %s is not configured' % meter_name))
            for port_no in acl.get_mirror_destinations():
                port = self.ports[port_no]
                port.output_only = True
            return acl.build(vid, self.meters)

        def verify_acl_exact_match(acls):
            for acl in acls:
                test_config_condition(acl.exact_match != acls[0].exact_match, (
                    'ACLs when used together must have consistent exact_match'))
            return acls[0].exact_match

        def resolve_acls(valve_cl):
            """Resolve config references in ACLs."""
            # TODO: move this config validation to ACL object.
            port_acl_enabled = valve_cl.STATIC_TABLE_IDS
            port_acl_matches = {}
            port_acl_set_fields = set()
            port_acl_exact_match = False
            port_acl_meter = False
            vlan_acl_matches = {}
            vlan_acl_exact_match = False
            vlan_acl_set_fields = set()
            vlan_acl_meter = False
            egress_acl_exact_match = False
            egress_acl_meter = False
            egress_acl_matches = {}
            egress_acl_set_fields = set()

            def merge_matches(matches, new_matches):
                for field, has_mask in list(new_matches.items()):
                    if has_mask or field not in matches:
                        matches[field] = has_mask

            for vlan in list(self.vlans.values()):
                if vlan.acls_in:
                    acls = []
                    for acl in vlan.acls_in:
                        matches, set_fields, meter = resolve_acl(acl, vlan.vid)
                        merge_matches(vlan_acl_matches, matches)
                        vlan_acl_set_fields = vlan_acl_set_fields.union(set_fields)
                        if meter:
                            vlan_acl_meter = True
                        acls.append(self.acls[acl])
                    vlan.acls_in = acls
                    vlan_acl_exact_match = verify_acl_exact_match(acls)
            for port in list(self.ports.values()):
                if port.acls_in:
                    test_config_condition(self.dp_acls, (
                        'dataplane ACLs cannot be used with port ACLs.'))
                    acls = []
                    for acl in port.acls_in:
                        matches, set_fields, meter = resolve_acl(acl, None)
                        merge_matches(port_acl_matches, matches)
                        port_acl_set_fields = port_acl_set_fields.union(set_fields)
                        if meter:
                            port_acl_meter = True
                        acls.append(self.acls[acl])
                    port.acls_in = acls
                    port_acl_exact_match = verify_acl_exact_match(acls)
                    port_acl_enabled = True
                if port.acls_out:
                    ''' ds180 - changes for acls_out'''
                    test_config_condition(self.dp_acls, (
                        'dataplane ACLs cannot be used with port ACLs.'))
                    aclso = []
                    for acl in port.acls_out:
                        matches, set_fields, meter = resolve_acl(acl, None)
                        merge_matches(egress_acl_matches, matches)
                        egress_acl_set_fields = egress_acl_set_fields.union(set_fields)
                        if meter:
                            egress_acl_meter = True
                        aclso.append(self.acls[acl])
                    port.acls_out = aclso
                    egress_acl_exact_match = verify_acl_exact_match(aclso)
                    egress_acl_enabled = True
            if self.dp_acls:
                acls = []
                for acl in self.acls:
                    matches, set_fields, meter = resolve_acl(acl, None)
                    merge_matches(port_acl_matches, matches)
                    port_acl_set_fields = port_acl_set_fields.union(set_fields)
                    if meter:
                        port_acl_meter = True
                    acls.append(self.acls[acl])
                self.dp_acls = acls
                port_acl_enabled = True
            if port_acl_enabled:
                port_acl_matches.update({'in_port': False})
            if egress_acl_enabled:
                egress_acl_matches.update({'in_port': False})
            port_acl_matches = {(field, mask) for field, mask in list(port_acl_matches.items())}
            vlan_acl_matches = {(field, mask) for field, mask in list(vlan_acl_matches.items())}

            # TODO: skip port_acl table if not configured.
            # TODO: dynamically configure output attribue
            override_table_config = {
                'port_acl': ValveTableConfig(
                    'port_acl',
                    exact_match=port_acl_exact_match,
                    meter=port_acl_meter,
                    output=True,
                    match_types=port_acl_matches,
                    set_fields=tuple(port_acl_set_fields)),
                'vlan_acl': ValveTableConfig(
                    'vlan_acl',
                    exact_match=vlan_acl_exact_match,
                    meter=vlan_acl_meter,
                    output=True,
                    match_types=vlan_acl_matches,
                    set_fields=tuple(vlan_acl_set_fields)),
                ''' ds180 - changes for acl_egress'''
                'acl_egress': ValveTableConfig(
                    'acl_egress',
                    exact_match=egress_acl_exact_match,
                    meter=egress_acl_meter,
                    output=True,
                    match_types=egress_acl_matches,
                    set_fields=tuple(egress_acl_set_fields)),
            }
            return override_table_config

        def resolve_vlan_names_in_routers():
            """Resolve VLAN references in routers."""
            dp_routers = {}
            for router_name, router in list(self.routers.items()):
                vlans = []
                for vlan_name in router.vlans:
                    vlan = resolve_vlan(vlan_name)
                    if vlan is not None:
                        vlans.append(vlan)
                if len(vlans) > 1:
                    dp_router = copy.copy(router)
                    dp_router.vlans = vlans
                    dp_routers[router_name] = dp_router
                vips = set()
                for vlan in vlans:
                    for vip in vlan.faucet_vips:
                        if (vip.ip in valve_packet.IPV4_LINK_LOCAL or
                                vip.ip in valve_packet.IPV6_LINK_LOCAL):
                            continue
                        vips.add(vip)
                for vip in vips:
                    for other_vip in vips - set([vip]):
                        test_config_condition(
                            vip.ip in other_vip.network,
                            'VIPs %s and %s overlap in router %s' % (
                                vip, other_vip, router_name))
            self.routers = dp_routers

        test_config_condition(not self.vlans, 'no VLANs referenced by interfaces in %s' % self.name)
        valve_cl = SUPPORTED_HARDWARE.get(self.hardware, None)
        test_config_condition(
            not valve_cl, 'hardware %s must be in %s' % (
                self.hardware, list(SUPPORTED_HARDWARE.keys())))

        for dp in dps:
            dp_by_name[dp.name] = dp
        for vlan in list(self.vlans.values()):
            vlan_by_name[vlan.name] = vlan
            if self.global_vlan:
                test_config_condition(
                    self.global_vlan == vlan.vid, 'VLAN %u is reserved by global_vlan' % vlan.vid)

        resolve_stack_dps()
        resolve_mirror_destinations()
        resolve_override_output_ports()
        resolve_vlan_names_in_routers()
        override_table_config = resolve_acls(valve_cl)

        # Only configure IP routing tables if enabled.
        ipvs = set()
        for vlan in list(self.vlans.values()):
            ipvs = ipvs.union(vlan.ipvs())
        for ipv in (4, 6):
            if ipv not in ipvs:
                table_name = 'ipv%u_fib' % ipv
                override_table_config[table_name] = ValveTableConfig(table_name)
        if not ipvs:
            override_table_config['vip'] = ValveTableConfig('vip')

        vlan_port_factor = len(self.vlans) * len(self.ports)
        self._configure_tables(override_table_config, valve_cl, vlan_port_factor)

        bgp_vlans = self.bgp_vlans()
        if bgp_vlans:
            for vlan in bgp_vlans:
                vlan_dps = [dp for dp in dps if vlan.vid in dp.vlans]
                test_config_condition(len(vlan_dps) != 1, (
                    'DPs %s sharing a BGP speaker VLAN is unsupported'))
            router_ids = {vlan.bgp_routerid for vlan in bgp_vlans}
            test_config_condition(len(router_ids) != 1, 'BGP router IDs must all be the same')
            bgp_ports = {vlan.bgp_port for vlan in bgp_vlans}
            test_config_condition(len(bgp_ports) != 1, 'BGP ports must all be the same')
            for vlan in bgp_vlans:
                test_config_condition(vlan.bgp_server_addresses != (
                    bgp_vlans[0].bgp_server_addresses), (
                        'BGP server addresses must all be the same'))

        for port in list(self.ports.values()):
            port.finalize()
        for vlan in list(self.vlans.values()):
            vlan.finalize()
        for acl in list(self.acls.values()):
            acl.finalize()
        for router in list(self.routers.values()):
            router.finalize()
        self.finalize()

    def get_native_vlan(self, port_num):
        """Return native VLAN for a port by number, or None."""
        if port_num in self.ports:
            return self.ports[port_num].native_vlan
        return None

    def bgp_vlans(self):
        """Return list of VLANs with BGP enabled."""
        return tuple([vlan for vlan in list(self.vlans.values()) if vlan.bgp_as])

    def dot1x_ports(self):
        """Return list of ports with 802.1x enabled."""
        return tuple([port for port in list(self.ports.values()) if port.dot1x])

    def to_conf(self):
        """Return DP config as dict."""
        result = super(DP, self).to_conf()
        if result is not None:
            if 'stack' in result:
                if result['stack'] is not None:
                    result['stack'] = {
                        'root_dp': str(self.stack['root_dp'])
                    }
            interface_dict = {}
            for port in list(self.ports.values()):
                interface_dict[port.name] = port.to_conf()
            result['interfaces'] = interface_dict
        return result

    def get_tables(self):
        """Return tables as dict for API call."""
        result = {}
        for table_name, table in list(self.tables.items()):
            result[table_name] = table.table_id
        return result

    def get_config_dict(self):
        """Return DP config as a dict for API call."""
        vlans_dict = {}
        for vlan in list(self.vlans.values()):
            vlans_dict[vlan.name] = vlan.to_conf()
        acls_dict = {}
        for acl_id, acl in list(self.acls.items()):
            acls_dict[acl_id] = acl.to_conf()
        return {
            'dps': {self.name: self.to_conf()},
            'vlans': vlans_dict,
            'acls': acls_dict}

    def _get_acl_config_changes(self, logger, new_dp):
        """Detect any config changes to ACLs.

        Args:
            logger (ValveLogger): logger instance.
            new_dp (DP): new dataplane configuration.
        Returns:
            changed_acls (dict): ACL ID map to new/changed ACLs.
        """
        changed_acls = {}
        for acl_id, new_acl in list(new_dp.acls.items()):
            if acl_id not in self.acls:
                changed_acls[acl_id] = new_acl
                logger.info('ACL %s new' % acl_id)
            else:
                if new_acl != self.acls[acl_id]:
                    changed_acls[acl_id] = new_acl
                    logger.info('ACL %s changed' % acl_id)
        return changed_acls

    def _get_vlan_config_changes(self, logger, new_dp):
        """Detect any config changes to VLANs.

        Args:
            logger (ValveLogger): logger instance.
            new_dp (DP): new dataplane configuration.
        Returns:
            changes (tuple) of:
                deleted_vlans (set): deleted VLAN IDs.
                changed_vlans (set): changed/added VLAN IDs.
        """
        deleted_vlans = set([])
        for vid in list(self.vlans.keys()):
            if vid not in new_dp.vlans:
                deleted_vlans.add(vid)

        changed_vlans = set([])
        for vid, new_vlan in list(new_dp.vlans.items()):
            if vid not in self.vlans:
                changed_vlans.add(vid)
                logger.info('VLAN %s added' % vid)
            else:
                old_vlan = self.vlans[vid]
                if old_vlan != new_vlan:
                    if not old_vlan.ignore_subconf(new_vlan):
                        changed_vlans.add(vid)
                        logger.info('VLAN %s config changed' % vid)
                else:
                    # Preserve current VLAN including current
                    # dynamic state like caches, if VLAN and ports
                    # did not change at all.
                    new_dp.vlans[vid].merge_dyn(old_vlan)

        if not deleted_vlans and not changed_vlans:
            logger.info('no VLAN config changes')

        return (deleted_vlans, changed_vlans)

    def _get_port_config_changes(self, logger, new_dp, changed_vlans, changed_acls):
        """Detect any config changes to ports.

        Args:
            logger (ValveLogger): logger instance.
            new_dp (DP): new dataplane configuration.
            changed_vlans (set): changed/added VLAN IDs.
            changed_acls (dict): ACL ID map to new/changed ACLs.
        Returns:
            changes (tuple) of:
                all_ports_changed (bool): True if all ports changed.
                deleted_ports (set): deleted port numbers.
                changed_ports (set): changed/added port numbers.
                changed_acl_ports (set): changed ACL only port numbers.
        """
        all_ports_changed = False
        changed_ports = set([])
        changed_acl_ports = set([])

        for port_no, new_port in list(new_dp.ports.items()):
            if port_no not in self.ports:
                # Detected a newly configured port
                changed_ports.add(port_no)
                logger.info('port %s added' % port_no)
            else:
                old_port = self.ports[port_no]
                # An existing port has configs changed
                if new_port != old_port:
                    # ACL optimization - did the ACL, and only the ACL change.
                    if old_port.ignore_subconf(new_port, ignore_keys=set(['acls_in'])):
                        if old_port.acls_in != new_port.acls_in:
                            changed_acl_ports.add(port_no)
                            old_acl_ids = old_port.acls_in
                            if old_acl_ids:
                                old_acl_ids = [acl._id for acl in old_acl_ids]
                            new_acl_ids = new_port.acls_in
                            if new_acl_ids:
                                new_acl_ids = [acl._id for acl in new_acl_ids]
                            logger.info('port %s ACL changed (ACL %s to %s)' % (
                                port_no, old_acl_ids, new_acl_ids))
                    else:
                        changed_ports.add(port_no)
                        logger.info('port %s reconfigured (%s)' % (
                            port_no, diff(old_port.to_conf(), new_port.to_conf(), context=1)))
                elif new_port.acls_in:
                    port_acls_changed = [acl for acl in new_port.acls_in if acl in changed_acls]
                    if port_acls_changed:
                        changed_acl_ports.add(port_no)
                        logger.info('port %s ACL changed (ACL %s content changed)' % (
                            port_no, port_acls_changed))

        # TODO: optimize case where only VLAN ACL changed.
        for vid in changed_vlans:
            for port in new_dp.vlans[vid].get_ports():
                changed_ports.add(port.number)

        deleted_ports = set(list(self.ports.keys())) - set(list(new_dp.ports.keys()))
        if deleted_ports:
            logger.info('deleted ports: %s' % deleted_ports)

        if changed_ports == set(new_dp.ports.keys()):
            all_ports_changed = True
        elif (not changed_ports and
              not deleted_ports and
              not changed_acl_ports):
            logger.info('no port config changes')

        return (all_ports_changed, deleted_ports,
                changed_ports, changed_acl_ports)

    def get_config_changes(self, logger, new_dp):
        """Detect any config changes.

        Args:
            logger (ValveLogger): logger instance
            new_dp (DP): new dataplane configuration.
        Returns:
            (tuple): changes tuple containing:

                deleted_ports (set): deleted port numbers.
                changed_ports (set): changed/added port numbers.
                changed_acl_ports (set): changed ACL only port numbers.
                deleted_vlans (set): deleted VLAN IDs.
                changed_vlans (set): changed/added VLAN IDs.
                all_ports_changed (bool): True if all ports changed.
        """
        def _table_configs(dp):
            return frozenset([
                table.table_config for table in list(dp.tables.values())])

        if self.ignore_subconf(new_dp):
            logger.info('DP base level config changed - requires cold start')
        elif _table_configs(self) != _table_configs(new_dp):
            logger.info('pipeline table config change - requires cold start')
        elif new_dp.routers != self.routers:
            logger.info('DP routers config changed - requires cold start')
        else:
            changed_acls = self._get_acl_config_changes(logger, new_dp)
            deleted_vlans, changed_vlans = self._get_vlan_config_changes(logger, new_dp)
            (all_ports_changed, deleted_ports,
             changed_ports, changed_acl_ports) = self._get_port_config_changes(
                 logger, new_dp, changed_vlans, changed_acls)
            return (deleted_ports, changed_ports, changed_acl_ports,
                    deleted_vlans, changed_vlans, all_ports_changed)
        # default cold start
        return (set(), set(), set(), set(), set(), True)


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from absl.testing import parameterized
import numpy
from tensorflow.contrib.distribute.python import mirrored_strategy as mirrored_lib
from tensorflow.contrib.optimizer_v2 import adagrad as adagrad_v2
from tensorflow.contrib.optimizer_v2 import gradient_descent as gradient_descent_v2
from tensorflow.python.distribute import combinations
from tensorflow.python.distribute import strategy_combinations
from tensorflow.python.distribute.single_loss_example import minimize_loss_example
from tensorflow.python.eager import context
from tensorflow.python.eager import test
from tensorflow.python.ops import control_flow_ops
from tensorflow.python.ops import variables

mirrored_strategy_with_gpu_and_cpu = combinations.NamedDistribution(
    "MirroredCPUAndGPU",
    lambda: mirrored_lib.MirroredStrategy(["/gpu:0", "/cpu:0"]),
    required_gpus=1)
mirrored_strategy_with_two_gpus = combinations.NamedDistribution(
    "Mirrored2GPUs",
    lambda: mirrored_lib.MirroredStrategy(["/gpu:0", "/gpu:1"]),
    required_gpus=2)

# pylint: disable=g-long-lambda
gradient_descent_optimizer_v2_fn = combinations.NamedObject(
    "GradientDescentV2", lambda: gradient_descent_v2.GradientDescentOptimizer(
        0.2))
adagrad_optimizer_v2_fn = combinations.NamedObject(
    "AdagradV2", lambda: adagrad_v2.AdagradOptimizer(0.001))

optimizers_v2 = [gradient_descent_optimizer_v2_fn, adagrad_optimizer_v2_fn]

def distributions_and_v2_optimizers():
  """DistributionStrategies and V2 Optimizers."""
  return combinations.combine(
      distribution=[
          strategy_combinations.one_device_strategy,
          strategy_combinations.mirrored_strategy_with_gpu_and_cpu,
          strategy_combinations.mirrored_strategy_with_two_gpus,
      ],
      optimizer_fn=optimizers_v2)

class MinimizeLossOptimizerV2Test(test.TestCase, parameterized.TestCase):

  @combinations.generate(
      combinations.times(
          distributions_and_v2_optimizers(),
          combinations.combine(mode=["graph"], use_callable_loss=[True, False])
          + combinations.combine(mode=["eager"], use_callable_loss=[True])))
  def testTrainNetwork(self, distribution, optimizer_fn,
                       use_callable_loss=True):
    with distribution.scope():
      optimizer = optimizer_fn()
      model_fn, dataset_fn, layer = minimize_loss_example(
          optimizer, use_bias=True, use_callable_loss=use_callable_loss)
      iterator = distribution.make_input_fn_iterator(lambda _: dataset_fn())

      def run_step():
        return control_flow_ops.group(
            distribution.experimental_local_results(
                distribution.extended.call_for_each_replica(
                    model_fn, args=(iterator.get_next(),))))

      if not context.executing_eagerly():
        with self.cached_session() as sess:
          sess.run(iterator.initialize())
          run_step = sess.make_callable(run_step())
        self.evaluate(variables.global_variables_initializer())

      weights, biases = [], []
      for _ in range(10):
        run_step()

        weights.append(self.evaluate(layer.kernel))
        biases.append(self.evaluate(layer.bias))

      error = abs(numpy.add(numpy.squeeze(weights), numpy.squeeze(biases)) - 1)
      is_not_increasing = all(y <= x for x, y in zip(error, error[1:]))
      self.assertTrue(is_not_increasing)

if __name__ == "__main__":
  test.main()


import cv2
import numpy as np
from matplotlib import pyplot as plt

def compara(S1, S2, D1, D2, D3):

    hist_S1 = cv2.calcHist([S1], [0], None, [256], [0, 256])
    hist_S2 = cv2.calcHist([S2], [0], None, [256], [0, 256])
    hist_D1 = cv2.calcHist([D1], [0], None, [256], [0, 256])
    hist_D2 = cv2.calcHist([D2], [0], None, [256], [0, 256])
    hist_D3 = cv2.calcHist([D3], [0], None, [256], [0, 256])

    #correl: mais proximo de 1
    #chisqr: mais proximo de 0
    #bhatta: mais proximo de 0

    Correl = np.array([cv2.compareHist(hist_S1, hist_S1, cv2.HISTCMP_CORREL), cv2.compareHist(hist_S1, hist_S2, cv2.HISTCMP_CORREL),
    cv2.compareHist(hist_S1, hist_D1, cv2.HISTCMP_CORREL), cv2.compareHist(hist_S1, hist_D2, cv2.HISTCMP_CORREL), cv2.compareHist(hist_S1, hist_D3, cv2.HISTCMP_CORREL)])

    Chisqr = np.array([cv2.compareHist(hist_S1, hist_S1, cv2.HISTCMP_CHISQR), cv2.compareHist(hist_S1, hist_S2, cv2.HISTCMP_CHISQR),
    cv2.compareHist(hist_S1, hist_D1, cv2.HISTCMP_CHISQR), cv2.compareHist(hist_S1, hist_D2, cv2.HISTCMP_CHISQR), cv2.compareHist(hist_S1, hist_D3, cv2.HISTCMP_CHISQR)])

    Bhatta = np.array([cv2.compareHist(hist_S1, hist_S1, cv2.HISTCMP_BHATTACHARYYA), cv2.compareHist(hist_S1, hist_S2, cv2.HISTCMP_BHATTACHARYYA),
    cv2.compareHist(hist_S1, hist_D1, cv2.HISTCMP_BHATTACHARYYA), cv2.compareHist(hist_S1, hist_D2, cv2.HISTCMP_BHATTACHARYYA), cv2.compareHist(hist_S1, hist_D3, cv2.HISTCMP_BHATTACHARYYA)])

    plt.figure()
    plt.suptitle("CORRELAÇÃO", fontsize = 20)
    plt.subplot(2, 5, 1), plt.imshow(S1, cmap='gray'), plt.title("%.5f" % (Correl[0])), plt.xticks([]), plt.yticks([])
    plt.subplot(2, 5, 2), plt.imshow(S2, cmap='gray'), plt.title("%.5f" % (Correl[1])), plt.xticks([]), plt.yticks([])
    plt.subplot(2, 5, 3), plt.imshow(D1, cmap='gray'), plt.title("%.5f" % (Correl[2])), plt.xticks([]), plt.yticks([])
    plt.subplot(2, 5, 4), plt.imshow(D2, cmap='gray'), plt.title("%.5f" % (Correl[3])), plt.xticks([]), plt.yticks([])
    plt.subplot(2, 5, 5), plt.imshow(D3, cmap='gray'), plt.title("%.5f" % (Correl[4])), plt.xticks([]), plt.yticks([])

    plt.figure()
    plt.suptitle("CHI-SQUARE", fontsize = 20)
    plt.subplot(2, 5, 1), plt.imshow(S1, cmap='gray'), plt.title("%.1f" % (Chisqr[0])), plt.xticks([]), plt.yticks([])
    plt.subplot(2, 5, 2), plt.imshow(S2, cmap='gray'), plt.title("%.1f" % (Chisqr[1])), plt.xticks([]), plt.yticks([])
    plt.subplot(2, 5, 3), plt.imshow(D1, cmap='gray'), plt.title("%.1f" % (Chisqr[2])), plt.xticks([]), plt.yticks([])
    plt.subplot(2, 5, 4), plt.imshow(D2, cmap='gray'), plt.title("%.1f" % (Chisqr[3])), plt.xticks([]), plt.yticks([])
    plt.subplot(2, 5, 5), plt.imshow(D3, cmap='gray'), plt.title("%.1f" % (Chisqr[4])), plt.xticks([]), plt.yticks([])

    plt.figure()
    plt.suptitle("BHATTACHARYYA", fontsize = 20)
    plt.subplot(2, 5, 1), plt.imshow(S1, cmap='gray'), plt.title("%.5f" % (Bhatta[0])), plt.xticks([]), plt.yticks([])
    plt.subplot(2, 5, 2), plt.imshow(S2, cmap='gray'), plt.title("%.5f" % (Bhatta[1])), plt.xticks([]), plt.yticks([])
    plt.subplot(2, 5, 3), plt.imshow(D1, cmap='gray'), plt.title("%.5f" % (Bhatta[2])), plt.xticks([]), plt.yticks([])
    plt.subplot(2, 5, 4), plt.imshow(D2, cmap='gray'), plt.title("%.5f" % (Bhatta[3])), plt.xticks([]), plt.yticks([])
    plt.subplot(2, 5, 5), plt.imshow(D3, cmap='gray'), plt.title("%.5f" % (Bhatta[4])), plt.xticks([]), plt.yticks([])

imagem1 = cv2.imread("trabalho7/cachorro.jpg")
imagem1 = cv2.cvtColor(imagem1, cv2.COLOR_BGR2GRAY)

imagem2 = cv2.imread("trabalho7/cachorro2.jpg")
imagem2 = cv2.cvtColor(imagem2, cv2.COLOR_BGR2GRAY)

imagem3 = cv2.imread("trabalho7/pato.jpg")
imagem3 = cv2.cvtColor(imagem3, cv2.COLOR_BGR2GRAY)

imagem4 = cv2.imread("trabalho7/dinossauro.jpg")
imagem4 = cv2.cvtColor(imagem4, cv2.COLOR_BGR2GRAY)

imagem5 = cv2.imread("trabalho7/macaco.jpg")
imagem5 = cv2.cvtColor(imagem5, cv2.COLOR_BGR2GRAY)

altura, largura = imagem2.shape

imagem1 = cv2.resize(imagem1, (largura, altura))
imagem3 = cv2.resize(imagem3, (largura, altura))
imagem4 = cv2.resize(imagem4, (largura, altura))
imagem5 = cv2.resize(imagem5, (largura, altura))

compara(imagem1, imagem2, imagem3, imagem4, imagem5)

plt.show()


import setuptools

with open("README.md", "r") as fh:
    long_description = fh.read()

setuptools.setup(
    name="gamma-facet", # Replace with your own username
    version="0.0.1",
    author="Konstantin Hemker",
    author_email="konstantin.hemker@googlemail.com",
    description="Mock package",
    long_description=long_description,
    long_description_content_type="text/markdown",
    url="https://github.com/konst-int-i/facet-mock",
    packages=setuptools.find_packages(),
    classifiers=[
        "Programming Language :: Python :: 3",
        "License :: OSI Approved :: MIT License",
        "Operating System :: OS Independent",
    ],
    python_requires='>=3.7',
)


import os
import numpy as np
from sklearn.preprocessing import MinMaxScaler
from sklearn.linear_model import LogisticRegression
D = np.loadtxt(os.path.join("ex2data1.txt"),delimiter=",")
print('Kích thước của tập dữ liệu: ', D.shape)
print('Lấy ra tập dữ liệu X, y')
X, y = D[:,:-1], D[:, -1]
print('Thực hiện MinMaxScaler')
#Khởi tạo bộ điều chỉnh dữ liệu
scaler = MinMaxScaler()
#Phải thực hiện thao tác fit(data) trước khi điều chỉnh dữ liệu
scaler.fit(X)
#Thực hiện điều chỉnh dữ liệu trên X, không điều chỉnh với y
X = scaler.transform(X)
print('Kích thước tập X: ', X.shape)
print('Kích thước vector y: ', y.shape)
print('Huấn luyện mô hình LogisticRegression')
#Khởi tạo mô hình
model = LogisticRegression()
#Huấn luyện mô hình với tập dữ liệu X, y
model.fit(X,y)
#Bộ trọng số tối ưu:
print('\t\tw optimal: ', model.coef_)


from typing import Annotated
from databases import Database
from fastapi import Depends

from database.db import CommonDB

class RoomService:
    def __init__(self, db: Database):
        self.db = db

    async def get_all(self):
        return await self.db.fetch_all("SELECT id, name, created_at FROM public.rooms")

    async def get_one(self, room_id: int):
        return await self.db.fetch_one(
            "SELECT id, name, created_at FROM public.rooms WHERE id = :id",
            values={"id": room_id},
        )

def get_room_service(db: CommonDB):
    return RoomService(db)

CommonRoomService = Annotated[RoomService, Depends(get_room_service, use_cache=True)]


#%%
import torchsummaryX
import torch
import argparse
from tqdm import tqdm

import numpy as np

from model.single_rnn_new import SingleRNN
from util.traj_dataset_general import TrajDatasetGeneral, wrap_data
from util.data_prepare import traj_prepare

parser = argparse.ArgumentParser(description='PyTorch MNIST Example')
parser.add_argument('--begin-index', type=int, default=6*60, metavar='N', help='--begin-index')
parser.add_argument('--time-interval', type=int, default=15, metavar='N', help='--time-interval')
parser.add_argument('--num-per-day', type=int, default=18*4, metavar='N', help='--num-per-day')
parser.add_argument('--batch-size', type=int, default=256, metavar='N', help='--batch-size')
parser.add_argument('--embed-size', type=int, default=40, metavar='N', help='--embed-size')
parser.add_argument('--epochs', type=int, default=10, metavar='N')
parser.add_argument('--hid-size', type=int, default=80, metavar='N', help='hidden size')
parser.add_argument('--hid-layer', type=int, default=2, metavar='N', help='hidden layer')
parser.add_argument('--drop-out', type=float, default=0.2, metavar='DR_R', help='drop out rate')
parser.add_argument('--lr', type=float, default=5e-3, metavar='LR', help='learning rate')

args, unknown = parser.parse_known_args()
torch.manual_seed(3)
np.random.seed(3)

source_folder = "/home/ubuntu/pflow_data/filtered"
traj_path = f"{source_folder}/traj.csv"
code_path = f"{source_folder}/code.csv"

num_samples = 50000
traj, n_traj, translator = traj_prepare(traj_path, code_path,
                                        num_samples,
                                        use_cols=args.begin_index+np.arange(args.num_per_day)*args.time_interval)
                                        #add_idx=["0/SOS"])

NUM_CLASS = n_traj
DATA_KEYS = ["data", "label"]

num_training_sample = int(num_samples * 0.8)
training_data, validation_data= traj[:num_training_sample, :-1], traj[num_training_sample:, :-1]
training_label, validation_label= traj[:num_training_sample, 1:], traj[num_training_sample:, 1:]

training_loader = torch.utils.data.DataLoader(
            TrajDatasetGeneral([training_data, training_label]),
            batch_size=args.batch_size,
            shuffle=True, num_workers=4,
            drop_last=False)
validation_loader = torch.utils.data.DataLoader(
            TrajDatasetGeneral([validation_data, validation_label]),
            batch_size=args.batch_size,
            shuffle=True, num_workers=4,
            drop_last=False)

DEVICE = torch.device("cuda:0")

sing_gru = SingleRNN(num_class=NUM_CLASS,
                     hid_size=args.hid_size, hid_layers=args.hid_layer,
                     drop_out_rate=args.drop_out, embedding_size=args.embed_size,
                     learning_rate=args.lr,
                     data_keys=DATA_KEYS,
                     writer_comment="Single_RNN")

test_dt = training_loader.dataset[0:10][0].T
test_dt = torch.tensor(test_dt)
model_summary = torchsummaryX.summary(sing_gru, test_dt)
#%%
sing_gru.to(DEVICE)

sing_gru.fit(training_loader=training_loader, validation_loader=validation_loader,
             epochs=args.epochs, device=DEVICE,
             optimizer=sing_gru.optimizer, criterion=sing_gru.criterion)
log_dir = sing_gru.writer.log_dir
sing_gru.writer.close()
torch.save(sing_gru, f"{log_dir}/model.pt")
#%%

data = training_loader.dataset.get_sample(15)
begin_time = int(args.num_per_day * 0.35)
data[0] = data[0][:, :begin_time]
data = wrap_data(data, keys=DATA_KEYS, device=DEVICE)
sampled_traj = sing_gru.sample_traj(data, predict_length=args.num_per_day-args.begin_index,
                                    sample_method="multinomial")


#!/usr/bin/env python3
import codecs
import os
import re
from setuptools import setup, find_packages

META_PATH = os.path.join("hostinet", "__version__.py")
HERE = os.path.abspath(os.path.dirname(__file__))
PACKAGES = find_packages()
KEYWORDS = ["hostinet", "sdk", "developers"]
CLASSIFIERS = [
    "Programming Language :: Python",
    "Development Status :: 5 - Production/Stable",
    "Intended Audience :: Developers",
    "Natural Language :: English",
    "License :: OSI Approved :: MIT License",
    "Operating System :: OS Independent",
    "Programming Language :: Python",
    "Programming Language :: Python :: 3",
    "Programming Language :: Python :: 3.5",
    "Programming Language :: Python :: 3.6",
    "Programming Language :: Python :: 3.7",
    "Programming Language :: Python :: Implementation :: CPython",
    "Programming Language :: Python :: Implementation :: PyPy",
    "Topic :: Software Development :: Libraries :: Python Modules",
]
INSTALL_REQUIRES = ['requests']

def read(*parts):
    """
    Build an absolute path from *parts* and and return the contents of the
    resulting file.  Assume UTF-8 encoding.
    """
    with codecs.open(os.path.join(HERE, *parts), "rb", "utf-8") as f:
        return f.read()

META_FILE = read(META_PATH)

def find_meta(meta):
    """
    Extract __*meta*__ from META_FILE.
    """
    meta_match = re.search(
        r"^__{meta}__ = ['\"]([^'\"]*)['\"]".format(meta=meta),
        META_FILE, re.M
    )
    if meta_match:
        return meta_match.group(1)
    raise RuntimeError("Unable to find __{meta}__ string.".format(meta=meta))

with open("README.md", "r") as fh:
    long_description = fh.read()

setup(
    author="Hostinet, SL.",
    author_email="desarrollo@hostinet.com",
    description="Hostinet - Software Development Kit for Python.",
    long_description=long_description,
    long_description_content_type="text/markdown",
    license=find_meta("license"),
    name="sdk-python",
    url="https://github.com/hostinet/sdk-python",
    version=find_meta("version"),
    classifiers=CLASSIFIERS,
    packages=PACKAGES,
    install_requires=INSTALL_REQUIRES,
    entry_points={}
)


#
# Author: Jeremy Pedersen (and ChatGPT)
# Updated: 2023-10-08
#
# Deletes one or more SageMaker notebook instances which match a pattern
#
# NOTE: Pattern matching is case-insensitive!
#
import boto3
import argparse

#############
# Functions #
#############

def delete_notebooks(region, pattern):
    # Initialize the boto3 client
    sagemaker = boto3.client('sagemaker', region_name=region)

    # Initialize the NextToken value
    next_token = None

    while True:
        # If NextToken is not None, include it in the request
        if next_token:
            notebook_instances = sagemaker.list_notebook_instances(NextToken=next_token)
        else:
            notebook_instances = sagemaker.list_notebook_instances()

        # Iterate over each instance
        for instance in notebook_instances['NotebookInstances']:
            instance_name = instance['NotebookInstanceName']

            if pattern.lower() in instance_name.lower():
                # Delete each notebook instance
                print(f'Deleting {instance_name}...')
                try:
                    sagemaker.delete_notebook_instance(NotebookInstanceName=instance_name)
                except:
                    print(f'Unable to delete instance {instance_name}')

        # If the response includes a NextToken, assign it to the variable for the next loop iteration
        if 'NextToken' in notebook_instances:
            next_token = notebook_instances['NextToken']
        else:
            # If there is no NextToken in the response, we've reached the end of the list
            break

##################
# The real stuff #
##################

# Initialize the argument parser
parser = argparse.ArgumentParser(description='A script to delete all SageMaker Notebooks in a given region')
parser.add_argument('-r', '--region', type=str, required=True, help='The AWS region to use (ex: us-west-1)')
parser.add_argument('-p', '--pattern', type=str, required=True, help='The pattern to match for notebook names (ex: "my-notebook-")')

# Parse the command line arguments
args = parser.parse_args()

print('Deleting notebook(s)...')
delete_notebooks(args.region, args.pattern)
print('Done!')


import pygame as p
import random

def setup():
    global screen, clock
    p.init()
    screen = p.display.set_mode((0, 0), p.FULLSCREEN)
    clock = p.time.Clock()

def main():
    done = False
    snowFlakes = []
    santaImg = p.image.load("images/reinder-image.jpg")  # it returns a Surface object
    scaleFactor = 0.75
    santaImg = p.transform.scale(santaImg, (int(santaImg.get_width() * scaleFactor),
                                            int(santaImg.get_height() * scaleFactor)))  # resize original image to 0.75X
    # santaImg = p.transform.scale(santaImg, (screen.get_width(),screen.get_height()*0.75))  # set image to size (screen_width , 0.75 * screen_height)
    santaX = -santaImg.get_width()
    santaVX = 30
    for i in range(500):
        x = random.randint(0, screen.get_width())
        y = random.randint(0, screen.get_height())
        r = random.randint(1, 5)
        snowFlakes.append((x, y, r))

    while not done:
        screen.fill(p.Color("lightskyblue"))
        # go through the event queue
        for e in p.event.get():
            if e.type == p.QUIT or (e.type == p.KEYDOWN and e.key == p.K_ESCAPE):
                done = True
        screen.blit(santaImg, (santaX, 100))
        if (santaX > screen.get_width() and santaVX > 0) or (santaX < -santaImg.get_width() and santaVX< 0):

            santaVX = -santaVX
            santaImg = p.transform.flip(santaImg, True, False)
            scaleFactor = 0.8
            santaImg = p.transform.scale(santaImg, (int(santaImg.get_width() * scaleFactor),
                                                    int(santaImg.get_height() * scaleFactor)))  # resize original image to 0.75X

        santaX += santaVX

        p.draw.rect(screen, p.Color("snow"),
                    p.Rect(0, 0.75 * screen.get_height(), screen.get_width(), 0.25 * screen.get_height()))
        for i in range(len(snowFlakes)):
            snowFlake = snowFlakes[i]
            snowFlakes[i] = (snowFlake[0] + random.randint(-1, 1), snowFlake[1] + random.randint(1, 4), snowFlake[2])

            p.draw.circle(screen, p.Color("snow2"), (snowFlake[0], snowFlake[1]), snowFlake[2])
            if snowFlake[1] > screen.get_height():
                snowFlakes[i] = (snowFlake[0], 0, snowFlake[2])  # reset the snowflake back to the top

        p.display.flip()  # redraws the display
        clock.tick(30)  # max out af 30 fps

setup()
main()


from app.services.api.entities.channel import Channel
from injector import inject
from app.services.api.channel import ChannelApiService
from app.bot_config.conversation_states_constants import CHANGE_NAME
from telegram import Update, InlineKeyboardButton, ParseMode, InlineKeyboardMarkup
from telegram.ext import ConversationHandler
from app.core.callback_context import CallbackContext
from app.transalation import trans as _

class NameConversation:
    @inject
    def __init__(self, channel_service: ChannelApiService):
        self.channel_service = channel_service

    def start(self, update: Update, context: CallbackContext) -> int:
        keyboard = [
            [
                InlineKeyboardButton(
                    text=_("back_in_history_btn"), callback_data="/myspace"
                ),
            ]
        ]
        update.effective_message.edit_text(
            _("channel_change_name"),
            parse_mode=ParseMode.MARKDOWN,
            reply_markup=InlineKeyboardMarkup(keyboard),
        )
        return CHANGE_NAME

    def save_name(self, update: Update, context: CallbackContext):
        text = update.effective_message.text
        channel: Channel = self.channel_service.get(context.user)
        channel.name = text
        self.channel_service.update(channel, user=context.user)
        keyboard = [
            [
                InlineKeyboardButton(
                    text=_("start_my_space"), callback_data="/myspace"
                ),
            ]
        ]
        update.effective_message.reply_text(
            _("channel_change_name_done") % text,
            parse_mode=ParseMode.MARKDOWN,
            reply_markup=InlineKeyboardMarkup(keyboard),
        )
        return ConversationHandler.END


from sqlalchemy import select
from sqlalchemy.orm import Session

from mmobot.db.models import Player

def recover_hp(engine):
    '''
    This job is scheduled to run at set time throughout the day to increment the HP
    of players over time.
    '''
    with Session(engine) as session:
        get_player_statement = (
            select(Player)
            .where(Player.is_active)
            .where(Player.hp < Player.max_hp)
        )
        for player in session.scalars(get_player_statement):
            player.hp += 1

        session.commit()


from django.shortcuts import render, HttpResponseRedirect, reverse
from .models import Lista, Produto
from django import forms

# Create your views here.

def index(request):
    return render(request, "listas/index.html", {
        "listas": Lista.objects.all()
    })

def lista(request, lista_id):
    lista = Lista.objects.get(id=lista_id)
    produto = lista.produto.all()
    sem_lista = Produto.objects.exclude(id=lista_id).all()
    return render(request, "listas/lista.html", {
        "lista": lista,
        "produto": produto,
        "sem_lista": sem_lista
    })

#def add(request):
#    if request.method == "POST":
#        form = Produto(request.POST)
#        if form.is_valid():
#            task = form.cleaned_data["task"]
#            request.session["tasks"] += [task]
#            return HttpResponseRedirect(reverse("listas:index"))
#        else:
#            return render(request, "listas/lista.html", {
#                "form": form
#            })
#
#    return render(request, "tasks/lista.html", {
#        "form": Produto()
#    })


from PIL import Image
from os import listdir
import os

###Change
width = 32
height = 32

srcDirectory = "pokemon_dataset_complete"
dstDirectory = "resized_datasets/" + str(width) + "x" + str(height)
imageList = listdir(srcDirectory)

errorDirectory = "error_log"
errorFile = open("resize_errors", 'w')
# nameList = [i.strip() for i in open("names.txt", 'r').readlines()]

# for imageFile in nameList:
# 	name = imageFile
# 	os.makedirs(dstDirectory + "/nearest/" + name)
# 	os.makedirs(dstDirectory + "/bilinear/" + name)
# 	os.makedirs(dstDirectory + "/bicubic/" + name)
# 	os.makedirs(dstDirectory + "/antialias/" + name)

for imageFile in imageList:

	im1 = Image.open(srcDirectory + "/" + imageFile)

	print (imageFile)

	try:
		im2 = im1.resize((width, height), Image.NEAREST)
		im3 = im1.resize((width, height), Image.BILINEAR)
		im4 = im1.resize((width, height), Image.BICUBIC)
		im5 = im1.resize((width, height), Image.ANTIALIAS)

		# name = ''.join([i for i in imageFile if not i.isdigit()])
		# name = name[0:-4]+"/"
		name = ""

		ext = ".png"
		im2.save(dstDirectory + "/nearest/" + name + imageFile)
		im3.save(dstDirectory + "/bilinear/" + name + imageFile)
		im4.save(dstDirectory + "/bicubic/" + name + imageFile)
		im5.save(dstDirectory + "/antialias/" + name + imageFile)

	except Exception:
			errorFile.write(imageFile + "\n")

other/Data Manipulation Augmentation and Pre-Procesing/resize_images.py


from django.conf.urls import url,include
from . import views
urlpatterns = [
	url(r'^$',views.index,name="index"),
	url('^check_game/$',views.check_game,name="check_game"),
	url('^create_game/$',views.create_game,name="create_game"),
	url('^join_game/$',views.join_game,name="join_game"),
	url('^start_game/$',views.start_game,name="start_game"),
	url('^game_arena/(?P<game_hash>[A-Za-z0-9\&\.\+_-]*)',views.game_arena,name="game_arena"),
	url('^check_win/$',views.check_win,name="check_win"),
	url('^update_winner/$',views.update_winner,name="update_winner"),
	url('^update_time/$',views.update_time,name="update_time"),
]


from model import Record
from pymysql import Connection

class RecordRepository:
    def __init__(self, conn: Connection) -> None:
        self.conn = conn
        self.cursor = self.conn.cursor()

    def get_record_by_id(self, id: int) -> Record:
        self.ping()
        sql = "SELECT `id`, `uid`, `path`, `request_time`, `finished_time`, `result` FROM `records` WHERE `id`=%s"
        self.cursor.execute(sql, (id,))
        row = self.cursor.fetchone()
        if row == None:
            return None
        return Record(
            id=row["id"], uid=row["uid"], path=row["path"],
            request_time=row["request_time"], finished_time=row["finished_time"],
            result=row["result"]
        )

    def get_records_by_uid(self, uid: int) -> list[Record]:
        self.ping()
        sql = "SELECT `id`, `uid`, `path`, `request_time`, `finished_time`, `result` FROM `records` WHERE `uid`=%s"
        self.cursor.execute(sql, (uid,))
        rows = self.cursor.fetchall()
        records = list()
        for row in rows:
            records.append(
                Record(
                    id=row["id"], uid=row["uid"], path=row["path"],
                    request_time=row["request_time"], finished_time=row["finished_time"],
                    result=row["result"]
                )
            )
        return records

    def create_record(self, record: Record) -> Record:
        self.ping()
        sql = "INSERT INTO `records` (`uid`, `path`, `request_time`, `finished_time`, `result`) VALUES (%s, %s, %s, %s, %s)"
        self.cursor.execute(sql, (record.uid, record.path, record.request_time, record.finished_time, record.result))
        self.conn.commit()
        return record

    def update_record(self, record: Record) -> Record:
        self.ping()
        sql = "UPDATE `records` SET `path`=%s, `request_time`=%s, `finished_time`=%s, `result`=%s WHERE `id`=%s"
        self.cursor.execute(sql, (record.path, record.request_time, record.finished_time, record.result, record.id))
        self.conn.commit()
        return record

    def delete_record(self, record: Record):
        self.ping()
        sql = "DELETE FROM `records` WHERE `id`=%s"
        self.cursor.execute(sql, (record.id))
        self.conn.commit()
        return

    def ping(self):
        self.conn.ping(reconnect=True)


import os
from pathlib import Path

import pandas as pd

def get_images_fps(imgs_root):
    img_df = pd.DataFrame(
        {"img_fp": list(map(str, Path(imgs_root).rglob("*.jpg")))},
    )
    img_df["is_dog"] = img_df["img_fp"].map(
        lambda fp: int("dog" in os.path.basename(fp).lower())
    )
    return img_df

if __name__ == "__main__":
    dataset_root = "cats_vs_dogs"
    train_root = os.path.join(dataset_root, "train")
    test_root = os.path.join(dataset_root, "test")

    splits_dest_dir = "data_splits"
    os.makedirs(splits_dest_dir, exist_ok=True)

    df_train = get_images_fps(train_root)
    df_test_split = get_images_fps(test_root)
    df_train = df_train.sample(frac=1, random_state=42).reset_index(drop=True)

    size = len(df_train)
    train_size = int(0.8 * size)
    df_train_split = df_train.iloc[:train_size]
    df_validation_split = df_train.iloc[train_size:]

    df_train_split.to_csv(os.path.join(splits_dest_dir, "train.csv"))
    df_validation_split.to_csv(os.path.join(splits_dest_dir, "validation.csv"))
    df_test_split.to_csv(os.path.join(splits_dest_dir, "test.csv"))


import glob
import logging
import os
import random
import sys
from pathlib import Path
import pickle

import numpy as np
from scipy import spatial, ndimage, misc
import torch

from lib.constants.dataset_sets import HEAD_CATS_SCANNET_200, COMMON_CATS_SCANNET_200, TAIL_CATS_SCANNET_200
from lib.dataset import VoxelizationDataset, DatasetPhase, str2datasetphase_type
from lib.pc_utils import read_plyfile, save_point_cloud
from lib.transforms import InstanceAugmentation
from lib.utils import read_txt, fast_hist, per_class_iu

from lib.constants.scannet_constants import *
from lib.datasets.preprocessing.utils import box_intersect

import MinkowskiEngine as ME

class ScannetVoxelizationDataset(VoxelizationDataset):

    # Voxelization arguments
    CLIP_BOUND = None
    TEST_CLIP_BOUND = None
    VOXEL_SIZE = 0.05

    # Load constants for label ids
    SCANNET_COLOR_MAP = SCANNET_COLOR_MAP_20
    CLASS_LABELS = CLASS_LABELS_20
    VALID_CLASS_IDS = VALID_CLASS_IDS_20

    # Augmentation arguments
    ROTATION_AUGMENTATION_BOUND = ((-np.pi / 64, np.pi / 64), (-np.pi / 64, np.pi / 64), (-np.pi,
                                                                                np.pi))
    TRANSLATION_AUGMENTATION_RATIO_BOUND = ((-0.2, 0.2), (-0.2, 0.2), (0, 0))
    ELASTIC_DISTORT_PARAMS = ((0.2, 0.4), (0.8, 1.6))

    ROTATION_AXIS = 'z'
    LOCFEAT_IDX = 2
    NUM_LABELS = max(SCANNET_COLOR_MAP_LONG.keys())  # Will be converted to 20 as defined in IGNORE_LABELS.
    IGNORE_LABELS = tuple(set(range(NUM_LABELS)) - set(VALID_CLASS_IDS))
    IS_FULL_POINTCLOUD_EVAL = True

    # If trainval.txt does not exist, copy train.txt and add contents from val.txt
    DATA_PATH_FILE = {
        DatasetPhase.Train: 'train.txt',
        DatasetPhase.Val: 'val.txt',
        DatasetPhase.TrainVal: 'trainval.txt',
        DatasetPhase.Test: 'test.txt'
    }

    def __init__(self,
                 config,
                 prevoxel_transform=None,
                 input_transform=None,
                 target_transform=None,
                 augment_data=True,
                 elastic_distortion=False,
                 cache=False,
                 phase=DatasetPhase.Train):
        if isinstance(phase, str):
            phase = str2datasetphase_type(phase)
        # Use cropped rooms for train/val
        data_root = config.scannet_path
        if phase not in [DatasetPhase.Train, DatasetPhase.TrainVal]:
            self.CLIP_BOUND = self.TEST_CLIP_BOUND
        data_paths = read_txt(os.path.join(data_root, self.DATA_PATH_FILE[phase]))
        logging.info('Loading {}: {}'.format(self.__class__.__name__, self.DATA_PATH_FILE[phase]))
        super().__init__(
            data_paths,
            data_root=data_root,
            prevoxel_transform=prevoxel_transform,
            input_transform=input_transform,
            target_transform=target_transform,
            ignore_label=config.ignore_label,
            return_transformation=config.return_transformation,
            augment_data=augment_data,
            elastic_distortion=elastic_distortion,
            config=config)

        # Load category weights for weighted CE and Focal
        self.category_weights = torch.ones(self.NUM_LABELS)
        category_weights_path = config.scannet_path + '/' + config.category_weights
        if os.path.isfile(category_weights_path):
            with open(category_weights_path, "rb") as input_file:
                category_weights = pickle.load(input_file)
                print('Loaded category weights for CE {}'.format(category_weights_path))

            for cat_id, cat_value in category_weights.items():
                if cat_id > 0:
                    mapped_id = self.label_map[cat_id]
                    self.category_weights[mapped_id] = cat_value

        # Load instance sampling weights for instance based balancing
        self.instance_sampling_weights = np.ones(self.NUM_LABELS)
        instance_sampling_weights_path = config.scannet_path + '/' + config.instance_sampling_weights
        if os.path.isfile(instance_sampling_weights_path) and config.sample_tail_instances:
            with open(instance_sampling_weights_path, "rb") as input_file:
                instance_sampling_weights = pickle.load(input_file)
                print('Loaded instance sampling probability weights {}'.format(instance_sampling_weights_path))
            for cat_id, cat_value in instance_sampling_weights.items():
                if cat_id > 0:
                    mapped_id = self.label_map[cat_id]
                    self.instance_sampling_weights[mapped_id] = cat_value
        self.instance_sampling_weights /= self.instance_sampling_weights.sum()

        # Precompute a mapping from ids to categories
        self.id2cat_name = {}
        for id, cat_name in zip(self.VALID_CLASS_IDS, self.CLASS_LABELS):
            self.id2cat_name[id] = cat_name

        # Load the bounding boxes from all instances of the dataset
        bb_path = config.scannet_path + '/' + config.bounding_boxes_path
        if os.path.isfile(bb_path):
            with open(bb_path, 'rb') as f:
                self.bounding_boxes = pickle.load(f)

        # To use instance level augmentation like color or scale shift
        self.instance_augmentation_transform = InstanceAugmentation(config)
        self.aug_color_prob = config.instance_augmentation_color_aug_prob
        self.aug_scale_prob = config.instance_augmentation_scale_aug_prob

        # Calculate head-common-tail ids
        self.head_ids = []
        self.common_ids = []
        self.tail_ids = []
        self.frequency_organized_cats = torch.zeros(self.NUM_LABELS, 3).bool()
        for scannet_id, scannet_cat in zip(self.VALID_CLASS_IDS, self.CLASS_LABELS):
            if scannet_cat in HEAD_CATS_SCANNET_200:
                self.head_ids += [self.label_map[scannet_id]]
                self.frequency_organized_cats[self.label_map[scannet_id], 0] = True
            elif scannet_cat in COMMON_CATS_SCANNET_200:
                self.common_ids += [self.label_map[scannet_id]]
                self.frequency_organized_cats[self.label_map[scannet_id], 1] = True
            elif scannet_cat in TAIL_CATS_SCANNET_200:
                self.tail_ids += [self.label_map[scannet_id]]
                self.frequency_organized_cats[self.label_map[scannet_id], 2] = True

    def add_instances_to_cloud(self, coords, feats, labels, scene_name, transformations):

        if self.config.is_train:
            phase = 'train'
        else:
            phase = 'val'

        coords = coords.astype(int)
        voxel_scale, trans_rot = transformations

        instance_folder = self.config.scannet_path + f'/train/{phase}_instances/'
        num_instances = self.config.num_instances_to_add
        samples = np.random.choice(self.VALID_CLASS_IDS, num_instances, p=self.instance_sampling_weights)
        scene_bbs = self.bounding_boxes[scene_name]

        # Get scene dimensions
        scene_maxes = np.amax(coords, axis=0)
        scene_mins = np.amin(coords, axis=0)
        scene_dims = scene_maxes - scene_mins + 1

        # Create height map of the scene
        height_map = np.zeros((scene_dims[0], scene_dims[1])) + scene_mins[2]
        def calculate_height(coord):
            height_map[coord[0], coord[1]] = max(coord[2], height_map[coord[0], coord[1]])
        mapped_coords = coords - [scene_mins[0], scene_mins[1], 0]
        [calculate_height(coord) for coord in mapped_coords]

        # Apply a max smoothing to the height map to fill holes
        filled_height_map = ndimage.maximum_filter(height_map, size=5)  # magic number with 2cm vx size = 10cm real

        # Add sample
        for sample in samples:
            sample_cat = self.id2cat_name[sample]
            cat_path = instance_folder + sample_cat
            file = cat_path + '/' + random.choice(os.listdir(cat_path))
            inst_coords, inst_feats, inst_labels, instance_ids, _ = self.load_ply_w_path(file, scene_name)

            # Add augmentation with attributes
            if self.config.instance_augmentation is not None:
                inst_labels = np.hstack((inst_labels[:, None], np.zeros_like(inst_labels)[:, None]))  # have it with all zeros, we will augment in latent space after encoding
            if self.config.instance_augmentation == 'raw':
                inst_coords, inst_feats, inst_labels = self.augment_instances(inst_coords, inst_feats, inst_labels, instance_ids) # do the augmentation here

            # Voxelize instance too
            inst_coords, inst_feats, inst_labels, inst_vox_transform = self.voxelizer.voxelize(
                inst_coords, inst_feats, inst_labels)

            # Get instance dimensions
            sample_maxes = np.amax(inst_coords, axis=0)
            sample_mins = np.amin(inst_coords, axis=0)
            sample_dims = sample_maxes - sample_mins + 1

            # Start finding a suitable location
            centroid = np.zeros(3, dtype=int)
            iter_num = 0
            while iter_num < self.config.max_instance_placing_iterations:

                # Add random BB
                random_x = random.randint(scene_mins[0], scene_maxes[0])
                random_y = random.randint(scene_mins[1], scene_maxes[1])
                height = float(filled_height_map[random_x - scene_mins[0], random_y - scene_mins[1]]) + 0  # Add some margin (+20cm)
                centroid = np.array([random_x, random_y, int(height + sample_dims[2] / 2.)])

                random_bb = np.array([centroid - (sample_dims / 2.0), centroid + (sample_dims / 2.0)])

                # Check intersection
                is_intersects = False
                for bb_dict in scene_bbs['instances']:
                    # Load and ransform BB
                    bb = np.copy(bb_dict['bb'])
                    homo_bb = np.hstack((bb, np.ones((bb.shape[0], 1), dtype=coords.dtype)))
                    bb = homo_bb @ voxel_scale.T[:, :3]

                    if box_intersect(bb, random_bb):
                        is_intersects = True
                        break

                if not is_intersects:
                    break

                iter_num += 1

            # Push point cloud to BB location
            inst_coords = inst_coords - np.mean(inst_coords, axis=0).astype(int) + centroid

            # Append new inputs
            coords = np.concatenate((coords, inst_coords))
            feats = np.concatenate((feats, inst_feats))
            labels = np.concatenate((labels, inst_labels))

        # Finally, apply rotation augmentation
        homo_coords = np.hstack((coords, np.ones((coords.shape[0], 1), dtype=coords.dtype)))
        coords_aug = np.floor(homo_coords @ trans_rot.T[:, :3])

        # Quantize again to remove touching points
        _, unique_map = ME.utils.sparse_quantize(coords_aug, return_index=True, ignore_label=self.config.ignore_label)
        coords_aug, feats, labels = coords_aug[unique_map], feats[unique_map], labels[unique_map]

        return coords_aug, feats, labels

    def augment_instances(self, coords, feats, labels, instance_ids=None):

        # To use precomputed inds we can only update the final scene cloud in the ends
        augmented_coords = []
        augmented_feats = []
        augmented_labels = []
        inds_to_remove = []

        scene_scale = np.array([(coords[:, 0].max() - coords[:, 0].min()),
                               (coords[:, 1].max() - coords[:, 1].min()),
                               (coords[:, 2].max() - coords[:, 2].min())])

        scene_cats = np.unique(labels)
        mapped_cats = np.vectorize(lambda x: self.label_map[x])(scene_cats)
        tail_cats = self.frequency_organized_cats[:, 2].cpu().numpy()
        common_cats = self.frequency_organized_cats[:, 1].cpu().numpy()
        pc_indexes = np.arange(coords.shape[0])

        for label_id in mapped_cats:
            # Only do for valid tail categories
            if label_id != self.ignore_mask and tail_cats[label_id]:

                # point inds for certain category
                cat_inds = labels[:, 0] == self.inverse_label_map[label_id]

                # Do this for scene instance augmentation
                if instance_ids is not None:
                    scene_instances = np.unique(instance_ids[cat_inds])
                    for inst in scene_instances:
                        p_inds = cat_inds * (instance_ids == inst)
                        inst_coords, inst_feats, inst_labels = coords[p_inds], feats[p_inds], labels[p_inds]

                        # Shift color with probability
                        if random.random() < self.aug_color_prob:
                            inst_coords, inst_feats, inst_labels = self.instance_augmentation_transform.shift_color(inst_coords, inst_feats, inst_labels)

                        # Shift scale with probability
                        elif random.random() < self.aug_scale_prob:
                            inst_coords, inst_feats, inst_labels = self.instance_augmentation_transform.shift_scale(inst_coords, inst_feats, inst_labels, scene_scale)

                        augmented_coords += [inst_coords]
                        augmented_feats += [inst_feats]
                        augmented_labels += [inst_labels]
                        inds_to_remove += [pc_indexes[p_inds][:, None]]

                else:  # sampled tail inst
                    inst_coords, inst_feats, inst_labels = coords, feats, labels

                    # Shift color with probability
                    if random.random() < self.aug_color_prob:
                        inst_coords, inst_feats, inst_labels = self.instance_augmentation_transform.shift_color(inst_coords, inst_feats, inst_labels)

                    # Shift scale with probability
                    elif random.random() < self.aug_scale_prob:
                        inst_coords, inst_feats, inst_labels = self.instance_augmentation_transform.shift_scale(inst_coords, inst_feats, inst_labels, scene_scale)

                    augmented_coords += [inst_coords]
                    augmented_feats += [inst_feats]
                    augmented_labels += [inst_labels]
                    inds_to_remove += [pc_indexes[:, None]]

        # Augmentation happened, update the clouds
        if len(augmented_coords) > 0:
            augmented_coords = np.vstack(augmented_coords)
            augmented_feats = np.vstack(augmented_feats)
            augmented_labels = np.vstack(augmented_labels)
            inds_to_remove = np.vstack(inds_to_remove).flatten()

            coords = np.delete(coords, inds_to_remove, axis=0)
            feats = np.delete(feats, inds_to_remove, axis=0)
            labels = np.delete(labels, inds_to_remove, axis=0)

            coords = np.vstack((coords, augmented_coords))
            feats = np.vstack((feats, augmented_feats))
            labels = np.vstack((labels, augmented_labels))

        return coords, feats, labels

    def __getitem__(self, index):

        coords, feats, labels, instance_ids, scene_name = self.load_ply(index)
        scene_name = scene_name.split('/')[-1].split('.')[0]

        # Downsample the pointcloud with finer voxel size before transformation for memory and speed
        if self.PREVOXELIZATION_VOXEL_SIZE is not None:
            _, inds = ME.utils.sparse_quantize(coords / self.PREVOXELIZATION_VOXEL_SIZE, return_index=True)
            coords = coords[inds]
            feats = feats[inds]
            labels = labels[inds]

        # Prevoxel transformations
        if self.prevoxel_transform is not None:
            coords, feats, labels = self.prevoxel_transform(coords, feats, labels)

        # Instance based augmentation
        if self.config.instance_augmentation is not None and self.augment_data:
            labels = np.hstack((labels[:, None], np.zeros_like(labels)[:, None]))
            if self.config.instance_augmentation == 'raw':
                coords, feats, labels = self.augment_instances(coords, feats, labels, instance_ids)

        # Add balanced instances
        if self.config.sample_tail_instances and self.augment_data:
            # Don't augment yet, but apply when everything is given
            coords, feats, labels, transformations = self.voxelizer.voxelize(coords, feats, labels, augment=False)
            coords, feats, labels = self.add_instances_to_cloud(coords, feats, labels, scene_name, transformations)
        else:
            # Voxelize as usual
            coords, feats, labels, transformations = self.voxelizer.voxelize(coords, feats, labels)

        # map labels not used for evaluation to ignore_label
        if self.input_transform is not None:
            coords, feats, labels = self.input_transform(coords, feats, labels)
        if self.target_transform is not None:
            coords, feats, labels = self.target_transform(coords, feats, labels)
        if self.IGNORE_LABELS is not None:
            mapper = lambda x: self.label_map[x]
            if labels.ndim == 1:
                labels = np.vectorize(mapper)(labels)
            else:
                labels[:, 0] = np.vectorize(mapper)(labels[:, 0])

        # Use coordinate features if config is set
        if self.AUGMENT_COORDS_TO_FEATS:
            coords, feats, labels = self._augment_coords_to_feats(coords, feats, labels)

        return_args = [coords, feats, labels, scene_name]

        if self.return_transformation:
            return_args.append(transformations[1].astype(np.float32))

        return tuple(return_args)

    def get_output_id(self, iteration):
        return '_'.join(Path(self.data_paths[iteration]).stem.split('_')[:2])

    def get_classids(self):
        return self.VALID_CLASS_IDS

    def get_classnames(self):
        return self.CLASS_LABELS

    def _augment_locfeat(self, pointcloud):
        # Assuming that pointcloud is xyzrgb(...), append location feat.
        pointcloud = np.hstack(
            (pointcloud[:, :6], 100 * np.expand_dims(pointcloud[:, self.LOCFEAT_IDX], 1),
             pointcloud[:, 6:]))
        return pointcloud

    def test_pointcloud(self, pred_dir, num_labels):

        print('Running full pointcloud evaluation.')
        eval_path = os.path.join(pred_dir, 'fulleval')
        os.makedirs(eval_path, exist_ok=True)
        # Join room by their area and room id.
        # Test independently for each room.
        sys.setrecursionlimit(100000)  # Increase recursion limit for k-d tree.
        hist = np.zeros((num_labels, num_labels))
        for i, data_path in enumerate(self.data_paths):
            room_id = self.get_output_id(i)
            pred = np.load(glob.glob(pred_dir + '/*pred*%04d.npy' % i)[0])

            # Scale with voxel size
            pred[:, :3] *= self.voxelizer.voxel_size

            # pred = np.load(os.path.join(pred_dir, 'pred_%04d_%02d.npy' % (i, 0)))

            # save voxelized pointcloud predictions
            save_point_cloud(
                np.hstack((pred[:, :3], np.array([self.SCANNET_COLOR_MAP[i] for i in pred[:, -1]]))),
                f'{eval_path}/{room_id}_voxel.ply',
                verbose=False)

            fullply_f = self.data_root / data_path
            query_pointcloud = read_plyfile(fullply_f)
            query_xyz = query_pointcloud[:, :3]
            query_label = query_pointcloud[:, -1]
            # Run test for each room.
            pred_tree = spatial.KDTree(pred[:, :3], leafsize=500)
            _, result = pred_tree.query(query_xyz)
            ptc_pred = pred[result, 3].astype(int)
            # Save prediciton in txt format for submission.
            np.savetxt(f'{eval_path}/{room_id}.txt', ptc_pred, fmt='%i')
            # Save prediciton in colored pointcloud for visualization.
            save_point_cloud(
                np.hstack((query_xyz, np.array([self.SCANNET_COLOR_MAP[i] for i in ptc_pred]))),
                f'{eval_path}/{room_id}.ply',
                verbose=False)
            # Evaluate IoU.
            if self.IGNORE_LABELS is not None:
                ptc_pred = np.array([self.label_map[x] for x in ptc_pred], dtype=np.int)
                query_label = np.array([self.label_map[x] for x in query_label], dtype=np.int)
            hist += fast_hist(ptc_pred, query_label, num_labels)
        ious = per_class_iu(hist) * 100
        print('mIoU: ' + str(np.nanmean(ious)) + '\n'
                                                 'Class names: ' + ', '.join(self.CLASS_LABELS) + '\n'
                                                                                'IoU: ' + ', '.join(
            np.round(ious, 2).astype(str)))

class ScannetVoxelization2cmDataset(ScannetVoxelizationDataset):
    VOXEL_SIZE = 0.02

class Scannet200VoxelizationDataset(ScannetVoxelizationDataset):
    # Load constants for label ids
    SCANNET_COLOR_MAP = SCANNET_COLOR_MAP_200
    CLASS_LABELS = CLASS_LABELS_200
    VALID_CLASS_IDS = VALID_CLASS_IDS_200

    NUM_LABELS = max(SCANNET_COLOR_MAP_LONG.keys()) + 1
    IGNORE_LABELS = tuple(set(range(NUM_LABELS)) - set(VALID_CLASS_IDS))

class Scannet200Voxelization2cmDataset(Scannet200VoxelizationDataset):
    VOXEL_SIZE = 0.02


import requests
import winsound
import random
import string
import urllib
import argparse

if __name__ == '__main__':
    p = argparse.ArgumentParser()
    p.add_argument('-a', dest='app_id')
    p.add_argument('-k', dest='app_key')
    p.add_argument('--tts_uri')
    p.add_argument('--tts_endpoint')
    args = p.parse_args()

    # Set your TTS Text here
    ttsText = "Hello World..."

    # Available Sources, http://dragonmobile.nuancemobiledeveloper.com/public/index.php?task=supportedLanguages
    ttsLang = "en_US"
    voice = "Susan"

    # Set a random ID via list comprehension, randomizing a-zA-Z and digits
    randomID = ''.join([random.choice(string.ascii_letters + string.digits) for n in range(8)])

    # HTTP Request Headers
    headers = {
        "Content-Type": "text/plain",
        "Accept": "audio/x-wav"
    }

    # Use Requests to send POST message with Query String
    params = urllib.urlencode({'appId': args.app_id, 'appKey': args.app_key, 'id': randomID, 'ttsLang': ttsLang,
                               'voice': voice})
    url = '%s%s?%s' % (args.tts_uri, args.tts_endpoint, params)
    req = requests.post(url, data=ttsText, headers=headers)

    # Play file using Windows
    play_wav = winsound.PlaySound(req.content, winsound.SND_MEMORY)

    # Play file using Linux
    # https://pypi.python.org/pypi/pysox/0.3.6.alpha
    # http://stackoverflow.com/questions/8195544/how-to-play-wav-data-right-from-memory


import pandas as pd
from itertools import count
from elasticsearch import Elasticsearch
from elasticsearch import RequestError,ConflictError
from credentials import es_url

class elasticSearchAPI():
    """
    defines methods to create an index and load or store data to it
    creating an instance of this class with an already existing index name
    uses the already created index with the passed name from elasticsearch
    """
    _autoincrement_id = count(0)
    created_indicies = {}
    def __init__(self,index_name,mapping=None):
        try:
            self.es = Elasticsearch(es_url)
            self.es.info()
        except ConnectionError as err:
            print(f"Connection to elastic search failed with message {err}")
        self.index_name = index_name
        self.document_id = self._autoincrement_id
        self.allowed_columns = list(mapping["properties"].keys())
        self.mapping = mapping

    def create_index(self):
        """
        create an index with a strict mapping with the named passed to the constructor
        """
        try:
            self.es.indices.create(index=self.index_name,mappings=self.mapping)
        except RequestError as err:
            if err.status_code ==400:
                print(f"inserting into existing index {self.index_name}")
            else:
                print(f"error creating index{err}")

    def store_reviews(self,reviews: pd.DataFrame):
        """
        stores a dataframe into the index passed to the constructor
        throws an error if the strict mapping defined in the constructor
        is not matched
        """
        data = reviews.to_dict(orient="list")
        try:
            self.create_index()
            self.es.create(index=self.index_name,id=self.document_id,document=data,refresh="wait_for")
        except ConflictError:
            print(f"the data with the columns {reviews.columns}was already inserted")
        except RequestError as err:
            print(f" allowed columns: {self.allowed_columns}")
            print(f"raw error msg {err}")

    def load_reviews(self) -> pd.DataFrame:
        response = self.es.search(index=self.index_name)
        response_data = response["hits"]["hits"]
        final_columns = {}
        if response["hits"]["total"]["value"]<=0:
            print("no results")
        else:
            data = response_data[0]["_source"]
            for name in self.allowed_columns:
                column = data[name]
                final_columns[name] = column
        return pd.DataFrame(final_columns)