text:
#!/usr/bin/python

##############################################################################################
# Copyright (C) 2014 Pier Luigi Ventre - (Consortium GARR and University of Rome "Tor Vergata")
# Copyright (C) 2014 Giuseppe Siracusano, Stefano Salsano - (CNIT and University of Rome "Tor Vergata")
# www.garr.it - www.uniroma2.it/netgroup - www.cnit.it
#
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Mininet Deployer.
#
# @author Pier Luigi Ventre <pl.ventre@gmail.com>
# @author Giuseppe Siracusano <a_siracusano@tin.it>
# @author Stefano Salsano <stefano.salsano@uniroma2.it>
#
#

from mininet.net import Mininet
import time
from mininet.cli import CLI
from mininet.node import RemoteController, Node, OVSKernelSwitch
from mininet.link import Link

from mininet.topo import SingleSwitchTopo
from mininet.log import setLogLevel, info, debug
from mininet.log import lg
from topo_parser import TopoParser

import networkx as nx
import numpy as np
import matplotlib.pyplot as plt

from oshi import *
from deployer_utils import *
from deployer_net_utils import *
from deployer_configuration_utils import *

from functools import partial
import subprocess
import os
import shutil
import sys
import argparse

aoshis = []
oshis = []
nets = []
switches = []
L2nets = []
ctrls = []
hosts = []

# XXX Parameter
# Vll path
vll_path = "" #"../vll_pusher_for_floodlights/"
# Executable path
path_quagga_exec = "" #"/usr/lib/quagga/"

# Controller Parameter
ctrls_ip = ['192.168.0.1']
ctrls_port = [6633]

# Parameter Coexistence Core Approach
CORE_APPROACH = 'A' # It can be A or B

# XXX Virtual Leased Line Configuration
LHS_tunnel = ['euh1', "euh2"]
RHS_tunnel = ['euh3', "euh1"]
tunnels = []
LHS_tunnel_aoshi = []
RHS_tunnel_aoshi = []
LHS_tunnel_port = []
RHS_tunnel_port = []
LHS_tunnel_vlan = []
RHS_tunnel_vlan = []
TRUNK_TO_TAG = {}
ACCESS_TO_TAG = {}
AOSHI_TO_TAG = {}

verbose = True

def check_tunnel_configuration():
	for i in range(0,len(LHS_tunnel)):
		host1 = LHS_tunnel[i]
		host2 = RHS_tunnel[i]
		if host1 not in hosts or host2 not in hosts:
			print "Error Misconfiguration Virtual Leased Line"
			print "Error Cannot Connect", host1, "To", host2
			sys.exit(2)

def SDN_tunnel_setup_oneside(net, side):
	global LHS_tunnel_aoshi
	global RHS_tunnel_aoshi
	global LHS_tunnel_port
	global RHS_tunnel_port
	global LHS_tunnel_vlan
	global RHS_tunnel_vlan
	k = 0
	if side == 'LHS':
		side_tunnel = LHS_tunnel
	else:
		side_tunnel = RHS_tunnel
	for host in side_tunnel:
		print "*** SDN Setup For", host
		nextHop = None
		i = 0
		while host not in L2nets[i].Nodes:
			i = i + 1
		if i == len(L2nets):
			print "Configuration Error"
			print "Cannot Find The Host", host
			sys.exit(-2)
		print "*** %s" % host, "is in %s" % L2nets[i].name
		done = False
		currentNode = host
		aoshi = L2nets[i].getAoshi(host)
		if side == 'RHS' and aoshi[0] == LHS_tunnel_aoshi[k] and aoshi[2] == LHS_tunnel_port[k]:
			print "*** Internal Tunnel"
			value = LHS_tunnel_vlan[k]
			tag = value
		else:
			default = 2
			# The Aoshi's link is the RHS
			value = AOSHI_TO_TAG.get(aoshi[2], default)
			tag = value
			AOSHI_TO_TAG[aoshi[2]] = value + 1
		print "*** VLAN Tag", tag
		while done == False:
			nextNode = L2nets[i].getNextHop(currentNode)
			if 'euh' in currentNode and 'sw' in nextNode[0]:
				rhs_new_link = net.getNodeByName(nextNode[0])
				lhs_new_link = net.getNodeByName(currentNode)
				l = net.addLink(lhs_new_link, rhs_new_link)
				tunnels[k].add_intf(l.intf1.name)
				# l.intf1 contains the NextHop's new port
				ACCESS_TO_TAG[l.intf2.name] = str(tag) + ","
				z = 0
				for network in nets:
					if len(network.belong(currentNode)) > 0:
						break
					z  = z + 1
				if z == len(nets):
					print "Configuration Error"
					print "Cannot Find The Host", currentNode, "In The OSPF Networks"
					sys.exit(-2)
				network.append_intf(l.intf1.name)
				currentNode = nextNode[0]
			elif 'sw' in currentNode:
				# NextNode[1] contains the currentNode's port
				default = ""
				tags = (TRUNK_TO_TAG.get(nextNode[1], default)).split(',')
				if str(tag) not in tags:
					TRUNK_TO_TAG[nextNode[1]] = TRUNK_TO_TAG.get(nextNode[1], default) + str(tag) + ","
				if 'aos' in nextNode[0]:
					aoshi = net.getNodeByName(nextNode[0])
					if side == 'LHS':
						LHS_tunnel_aoshi.append(aoshi.name)
						LHS_tunnel_port.append(nextNode[2])
						LHS_tunnel_vlan.append(tag)
					else:
						RHS_tunnel_aoshi.append(aoshi.name)
						RHS_tunnel_port.append(nextNode[2])
						RHS_tunnel_vlan.append(tag)
					done = True
				elif 'sw' in nextNode[0]:
					# NextNode[1] contains the nextHop's port
					tags = (TRUNK_TO_TAG.get(nextNode[2], default)).split(',')
					if str(tag) not in tags:
						TRUNK_TO_TAG[nextNode[2]] = TRUNK_TO_TAG.get(nextNode[2], default) + str(tag) + ","
				currentNode = nextNode[0]
			else:
				print "Error In The Network Configuration"
				print "Tunnel Setup Cannot Work Properly"
				sys.exit(-2)
		k = k + 1

def SDN_tunnel_setup(net):
	print "*** L2 Access Networks SDN Setup"
	print "*** SDN Setup For LHS"
	SDN_tunnel_setup_oneside(net, 'LHS')
	print "*** SDN Setup For RHS"
	SDN_tunnel_setup_oneside(net, 'RHS')

def IP_tunnel_setup():
	print "*** L2 Access Networks IP Setup"
	for host in hosts:
		print "*** IP Setup For", host
		nextHop = None
		i = 0
		while host not in L2nets[i].Nodes:
			i = i + 1
		if i == len(L2nets):
			print "Configuration Error"
			print "Cannot Find The Host", host
			sys.exit(-2)
		print "*** %s" % host, "is in %s" % L2nets[i].name
		done = False
		currentNode = host
		tag = L2nets[i].VlanIP
		while done == False:
			nextNode = L2nets[i].getNextHop(currentNode)
			if 'euh' in currentNode and 'sw' in nextNode[0]:
				# NextNode[1] contains the Link's LHS
				# ACCESS PORT is the RHS
				ACCESS_TO_TAG[nextNode[2]] = tag + ","
				currentNode = nextNode[0]
			elif 'sw' in currentNode:
				# NextNode[1] contains the Link's LHS
				default = ""
				tags = (TRUNK_TO_TAG.get(nextNode[1], default)).split(',')
				if tag not in tags:
					TRUNK_TO_TAG[nextNode[1]] = TRUNK_TO_TAG.get(nextNode[1], default) + tag + ","
				if 'aos' in nextNode[0]:
					done = True
				elif 'sw' in nextNode[0]:
					tags = (TRUNK_TO_TAG.get(nextNode[2], default)).split(',')
					if tag not in tags:
						TRUNK_TO_TAG[nextNode[2]] = TRUNK_TO_TAG.get(nextNode[2], default) + tag + ","
				currentNode = nextNode[0]
			else:
				print "Error In The Network Configuration"
				print "Tunnel Setup Cannot Work Properly"
				sys.exit(-2)

def configure_l2_accessnetwork():
	print "*** Configure L2 Access Networks"
	root = Node( 'root', inNamespace=False )
	print "*** Configure L2 Access Ports"
	for key, value in ACCESS_TO_TAG.iteritems():
		print "*** Configure", key, "As Access Port, TAG=", value
		root.cmd("ovs-vsctl set port %s tag=%s" %(key, value))
	print "*** Configure L2 Trunk Ports"
	for key, value in TRUNK_TO_TAG.iteritems():
		print "*** Configure", key, "As Trunk Port, TAG=", value
		root.cmd("ovs-vsctl set port %s trunks=%s" %(key, value))

def create_access_network(net):
	print "*** Create Access Networks"
	global nets
	global oshis
	global aoshis
	OSHI = len(oshis)
	for i in range(OSHI, (2*OSHI)):
		aoshi = (net.addHost('aos%s' % (i+1), loopback = give_me_next_loopback()))
		l = net.addLink(aoshi, oshis[i % OSHI])
		nets.append(OSPFNetwork(intfs=[l.intf1.name,l.intf2.name], ctrl=False))
		print "*** Connect", aoshi, "To", oshis[i % OSHI]
		create_l2_access_network(aoshi, net)
		aoshis.append(aoshi)

def create_l2_access_network(aoshi, net, n_host=1):
	global switches
	global hosts
	global L2nets
	name = "L2AccessNetwork" + str(len(L2nets) + 1);
	l2net = L2AccessNetwork(name, classification = 'B')
	print "*** Create L2 Access Network For", aoshi.name
	intfs = []
	hosts_in_rn = []
	print "*** Create L2 Switch"
	next = len(switches)
	sw = net.addSwitch("sw%s" % (next+1))
	print "*** Create Switch", sw.name
	hosts_in_rn.append(sw)
	l = net.addLink(sw, aoshi)
	print "*** Connect", sw, "To", aoshi
	l2net.addLink(l)
	intfs.append(l.intf2.name)
	switches.append(sw)

	# Create Another L2 Switch
	# temp = sw
	# next = len(switches)
	# print "*** Create Switch", sw.name
	# sw = net.addSwitch("sw%s" % (next+1))
	# hosts_in_rn.append(sw)
	# l = net.addLink(sw, temp)
	# print "*** Connect", sw, "To", temp
	# l2net.addLink(l)
	# switches.append(sw)

	print "*** Create End User Hosts"
	for i in range(len(hosts), (len(hosts) + n_host)):
		host = net.addHost(('euh%s') % (i+1), loopback = "0.0.0.0")
		l = net.addLink(host,sw)
		l2net.addLink(l)
		print "*** Connect", host, "To", sw
		hosts.append(host.name)
		intfs.append(l.intf1.name)
	nets.append(OSPFNetwork(intfs, ctrl=False, hello_int=2))
	fixIntf(hosts_in_rn)
	L2nets.append(l2net)

def buildTopoFromFile(param):
	global oshis
	global aoshis
	global switches
	global hosts
	global ctrls

	if verbose:
		print "*** Build Topology From Parsed File"
	parser = TopoParser(param, verbose=False)
	(ppsubnets, l2subnets) = parser.getsubnets()
	set_oshis = parser.oshis
	set_aoshis = parser.aoshis
	set_l2sws = parser.l2sws
	set_euhs = parser.euhs
	hosts_in_rn = []
	net = Mininet( controller=RemoteController, switch=OVSKernelSwitch, host=OSHI, build=False )
	if verbose:
		print "*** Build OSHI"
	for oshi in set_oshis:
		osh = net.addHost(oshi, loopback = give_me_next_loopback())
		oshis.append(osh)
	if verbose:
		print "*** Build AOSHI"
	for aoshi in set_aoshis:
		aos = net.addHost(aoshi, loopback = give_me_next_loopback())
		aoshis.append(aos)
	if verbose:
		print "*** Build L2SWS"
	for l2sw in set_l2sws:
		sw = net.addSwitch(l2sw)
		switches.append(sw)
		hosts_in_rn.append(sw)
	if verbose:
		print "*** Build EUHS"
	for euh in set_euhs:
		net.addHost(euh, loopback = "0.0.0.0")
		hosts.append(euh)
	if verbose:
		print "*** Create Core Networks Point To Point"
	i = 0
	for ppsubnet in ppsubnets:
		if ppsubnet.type == "CORE":
			if verbose:
				print "*** Subnet: Node %s - Links %s" %(ppsubnet.nodes, ppsubnet.links)
			node1 = net.getNodeByName(ppsubnet.links[0][0])
			node2 = net.getNodeByName(ppsubnet.links[0][1])
			l = net.addLink(node1, node2)
			nets.append(OSPFNetwork(intfs=[l.intf1.name,l.intf2.name], ctrl=False))
			if verbose:
				print "*** Connect", node1.name, "To", node2.name
		i = i + 1
	if verbose:
		print "*** Create Core Networks Switched"
	for l2subnet in l2subnets:
		if l2subnet.type == "CORE":
			if verbose:
				print "*** Subnet: Node %s - Links %s" % (ppsubnet.nodes, ppsubnet.links)
			intfs = []
			for link in l2subnet.links:
				node1 = net.getNodeByName(link[0])
				node2 = net.getNodeByName(link[1])
				l = net.addLink(node1, node2)
				if verbose:
					print "*** Connect", node1.name, "To", node2.name
				if 'sw' not in link[0] and 'sw' in link[1]:
					intfs.append(l.intf1.name)
				elif 'sw' in link[0] and 'sw' not in link[1]:
					intfs.append(l.intf2.name)
				elif 'sw' in link[0] and 'sw' in link[1]:
					continue
				else:
					print "Error Switched Networks - Both EndPoint != SW"
					sys.exit(-2)
			nets.append(OSPFNetwork(intfs, ctrl=False))
		i = i + 1
	if verbose:
		print "*** Create Access Networks Point To Point"
	i = 0
	for ppsubnet in ppsubnets:
		if ppsubnet.type == "ACCESS":
			# XXX The Order now is important
			"""if verbose:
				print "*** Subnet: Node %s - Links %s" %(ppsubnet.nodes, ppsubnet.links)
			node1 = net.getNodeByName(ppsubnet.links[0][0])
			node2 = net.getNodeByName(ppsubnet.links[0][1])
			l = net.addLink(node1, node2)
			nets.append(OSPFNetwork(intfs=[l.intf1.name,l.intf2.name], ctrl=False))
			if verbose:
				print "*** Connect", node1.name, "To", node2.name"""
			print "Error Not Managed For Now"
			sys.exit(-2)
		i = i + 1
	if verbose:
		print "*** Create Acces Networks Switched"
	for l2subnet in l2subnets:
		if l2subnet.type == "ACCESS":
			name = "L2AccessNetwork" + str(len(L2nets) + 1);
			l2net = L2AccessNetwork(name, classification = 'B')
			if verbose:
				print "*** Subnet: Node %s - Links %s" % (l2subnet.nodes, l2subnet.links)
				print "*** Create L2 Access Network - Classification", l2net.classification
			intfs = []
			# XXX The Order now is important
			ord_links = l2subnet.getOrderedLinks()
			for link in ord_links:
				node1 = net.getNodeByName(link[0])
				node2 = net.getNodeByName(link[1])
				l = net.addLink(node1, node2)
				l2net.addLink(l)
				if verbose:
					print "*** Connect", node1.name, "To", node2.name
				if 'sw' not in link[0] and 'sw' in link[1]:
					intfs.append(l.intf1.name)
				elif 'sw' in link[0] and 'sw' not in link[1]:
					intfs.append(l.intf2.name)
				elif 'sw' in link[0] and 'sw' in link[1]:
					continue
				else:
					print "Error Switched Networks - Both EndPoint != SW"
					sys.exit(-2)
			nets.append(OSPFNetwork(intfs, ctrl=False))
			L2nets.append(l2net)
		i = i + 1

	print "*** Creating controller"
	c1 = RemoteController( 'c1', ip=ctrls_ip[0], port=ctrls_port[0])
	ctrls.append(c1)
	hosts_in_rn.append(c1)

	# Connect the controller to the network
	print "*** Connect", osh.name, "To Controller"
	l = net.addLink(osh, c1)
	nets.append(OSPFNetwork(intfs=[l.intf1.name, l.intf2.name], ctrl=True))

	# Only needed for hosts in root namespace
	fixIntf(hosts_in_rn)

	# Utility function
	check_tunnel_configuration()

	for i in range(0, len(LHS_tunnel)):
		tunnels.append(Tunnel())

	print "*** Tunnels LHS:", LHS_tunnel
	print "*** Tunnels RHS:", RHS_tunnel

	# Tunnels Setup
	IP_tunnel_setup()
	SDN_tunnel_setup(net)

	i = 0
	for tunnel in tunnels :
		print "*** Tunnel %d, Subnet %s.%s.%s.%s, Intfs %s" % (i+1, tunnel.subnet[0], tunnel.subnet[1], tunnel.subnet[2], 0, tunnel.intfs)
		i = i + 1

	i = 0
	for l2net in L2nets:
		print "***", l2net.name
		print "*** Nodes:", l2net.Nodes
		print "*** Links:", l2net.Links
		print "*** Intfs:", l2net.intfs
		i = i + 1

	print "*** AOSHI Tag:", AOSHI_TO_TAG
	print "*** Trunk Port Configuration:", TRUNK_TO_TAG
	print "*** Access Port Configuration:", ACCESS_TO_TAG
	print "*** LHS AOSHI:", LHS_tunnel_aoshi
	print "*** RHS AOSHI:", RHS_tunnel_aoshi
	print "*** LHS Port:", LHS_tunnel_port
	print "*** RHS Port:", RHS_tunnel_port

	for network in nets:
		print "*** OSPF Network: %s.%s.%s.%s" % (network.subnet[0], network.subnet[1], network.subnet[2], 0), str(network.intfs) + ",", "cost %s," % network.cost, "hello interval %s," % network.hello_int
	return net

def Mesh(OSHI_n=4):
	global ctrls
	global oshis
	global aoshis
	global SDN_PORTS
	global TUNNEL_SETUP
	"Create A Mesh Topo"
	print "*** Mesh With", OSHI_n, "OSHI"
	"Creating OSHI"
	net = Mininet( controller=RemoteController, switch=OVSKernelSwitch, host=OSHI, build=False )
	i = 0
	h = 0
	print "*** Create Core Networks"
	for i in range(OSHI_n):
		oshi = (net.addHost('osh%s' % (i+1), loopback = give_me_next_loopback()))
		for rhs in oshis:
			l = net.addLink(oshi, rhs)
			nets.append(OSPFNetwork(intfs=[l.intf1.name,l.intf2.name], ctrl=False))
			print "*** Connect", oshi, "To", rhs
		oshis.append(oshi)

	print "*** Creating controller"
	#c0 = net.addController( 'c0', ip=ctrl_root_ip, port=ctrl_root_port )
	c1 = RemoteController( 'c1', ip=ctrls_ip[0], port=ctrls_port[0])
	ctrls.append(c1)

	hosts_in_rn = []
	hosts_in_rn.append(c1)
	# Connect the controller to the network
	print "*** Connect", oshi.name, "To Controller"
	l = net.addLink(oshi, c1)
	nets.append(OSPFNetwork(intfs=[l.intf1.name, l.intf2.name], ctrl=True))

	# Only needed for hosts in root namespace
	fixIntf(hosts_in_rn)

	# Utility function
	create_access_network(net)

	# Utility function
	check_tunnel_configuration()

	for i in range(0, len(LHS_tunnel)):
		tunnels.append(Tunnel())

	print "*** Tunnels LHS:", LHS_tunnel
	print "*** Tunnels RHS:", RHS_tunnel

	# Tunnels Setup
	IP_tunnel_setup()
	SDN_tunnel_setup(net)

	i = 0
	for tunnel in tunnels :
		print "*** Tunnel %d, Subnet %s.%s.%s.%s, Intfs %s" % (i+1, tunnel.subnet[0], tunnel.subnet[1], tunnel.subnet[2], 0, tunnel.intfs)
		i = i + 1

	i = 0
	for l2net in L2nets:
		print "***", l2net.name
		print "*** Nodes:", l2net.Nodes
		print "*** Links:", l2net.Links
		print "*** Intfs:", l2net.intfs
		i = i + 1

	print "*** AOSHI Tag:", AOSHI_TO_TAG
	print "*** Trunk Port Configuration:", TRUNK_TO_TAG
	print "*** Access Port Configuration:", ACCESS_TO_TAG
	print "*** LHS AOSHI:", LHS_tunnel_aoshi
	print "*** RHS AOSHI:", RHS_tunnel_aoshi
	print "*** LHS Port:", LHS_tunnel_port
	print "*** RHS Port:", RHS_tunnel_port
	for network in nets:
		print "*** OSPF Network: %s.%s.%s.%s" % (network.subnet[0], network.subnet[1], network.subnet[2], 0), str(network.intfs) + ",", "cost %s," % network.cost, "hello interval %s," % network.hello_int
	return net

def erdos_renyi_from_nx(n, p):
	g = nx.erdos_renyi_graph(n,p)
	global ctrls
	global oshis
	global aoshis
	global SDN_PORTS
	global TUNNEL_SETUP
	"Create An Erdos Reny Topo"
	"Creating OSHI"
	net = Mininet( controller=RemoteController, switch=OVSKernelSwitch, host=OSHI, build=False )
	i = 0
	h = 0
	# This is the basic behavior, with nx we create only the core network
	for n in g.nodes():
		n = n + 1
		oshi = (net.addHost('osh%s' % (n), loopback = give_me_next_loopback()))
		oshis.append(oshi)
	for (n1, n2) in g.edges():
		n1 = n1 + 1
		n2 = n2 + 1
		lhs = net.getNodeByName('osh%s' % n1)
		rhs = net.getNodeByName('osh%s' % n2)
		l = net.addLink(lhs, rhs)
		nets.append(OSPFNetwork(intfs=[l.intf1.name,l.intf2.name], ctrl=False))
		print "*** Connect", lhs, "To", rhs

	hosts_in_rn = []
	c1 = RemoteController( 'c1', ip=ctrls_ip[0], port=ctrls_port[0])
	ctrls.append(c1)
	hosts_in_rn.append(c1)

	# Connecting the controller to the network
	print "*** Connect %s" % oshi," To c1"
	l = net.addLink(oshi, c1)
	nets.append(OSPFNetwork(intfs=[l.intf1.name,l.intf2.name], ctrl=True, hello_int=5))

	# Only needed for hosts in root namespace
	fixIntf(hosts_in_rn)

	# Utility function
	create_access_network(net)

	# Utility function
	check_tunnel_configuration()

	for i in range(0, len(LHS_tunnel)):
		tunnels.append(Tunnel())

	print "*** Tunnels LHS:", LHS_tunnel
	print "*** Tunnels RHS:", RHS_tunnel

	# Tunnels Setup
	IP_tunnel_setup()
	SDN_tunnel_setup(net)

	i = 0
	for tunnel in tunnels :
		print "*** Tunnel %d, Subnet %s.%s.%s.%s, Intfs %s" % (i+1, tunnel.subnet[0], tunnel.subnet[1], tunnel.subnet[2], 0, tunnel.intfs)
		i = i + 1

	i = 0
	for l2net in L2nets:
		print "***", l2net.name
		print "*** Nodes:", l2net.Nodes
		print "*** Links:", l2net.Links
		print "*** Intfs:", l2net.intfs
		i = i + 1

	print "*** AOSHI Tag:", AOSHI_TO_TAG
	print "*** Trunk Port Configuration:", TRUNK_TO_TAG
	print "*** Access Port Configuration:", ACCESS_TO_TAG
	print "*** LHS AOSHI:", LHS_tunnel_aoshi
	print "*** RHS AOSHI:", RHS_tunnel_aoshi
	print "*** LHS Port:", LHS_tunnel_port
	print "*** RHS Port:", RHS_tunnel_port

	for network in nets:
		print "*** OSPF Network: %s.%s.%s.%s" % (network.subnet[0], network.subnet[1], network.subnet[2], 0), str(network.intfs) + ",", "cost %s," % network.cost, "hello interval %s," % network.hello_int

	# We generate the topo's png
	pos = nx.circular_layout(g)
        nx.draw(g, pos)
        plt.savefig("topo.png")

	return net

def buildTopoFromNx(topo, args):
	if topo == 'e_r':
		data = args.split(",")
		args = []
		args.append(int(data[0]))
		args.append(float(data[1]))
		if len(args) >= 2:
			if args [0] > 10 or args[1] > 1:
				print "Warning Parameter Too High For Erdos Renyi", "Nodes %s" % args[0], "Interconnection Probability %s" % args[1]
				print "Using Default Parameter"
				args[0] = 5
				args[1] = 0.8
		else :
			args[0] = 5
			args[1] = 0.8
		print "Erdos Renyi", "Nodes %s" % args[0], "Interconnection Probability %s" % args[1]
		return erdos_renyi_from_nx(args[0], args[1])

	print "Error NX Wrong Parameter"
	sys.exit(-2)

def configure_env_oshi(oshi):
	global next_ctrl
	print "*** Configuring Environment For", oshi.name
	shutil.rmtree("/tmp/" + oshi.name, ignore_errors=True)
	os.mkdir("/tmp/" + oshi.name)
	ctrl_ip = ctrls_ip[next_ctrl]
	ctrl_port = ctrls_port[next_ctrl]
	next_ctrl = (next_ctrl + 1) % len(ctrls_ip)
	configure_ovs(oshi, ctrl_ip, ctrl_port)
	configure_quagga(oshi)
	oshi.cmd("echo 1 > /proc/sys/net/ipv4/ip_forward ")
	oshi.cmd("echo 0 > /proc/sys/net/ipv4/conf/all/rp_filter")
	for intf in oshi.nameToIntf:
		cmd = "echo 0 > /proc/sys/net/ipv4/conf/" + intf + "/rp_filter"
		oshi.cmd(cmd)
		intf = "vi%s" % (strip_number(intf))
		if CORE_APPROACH == 'A':
			VLAN_IP = 1
			intf = intf + "." + str(VLAN_IP)
		cmd = "echo 0 > /proc/sys/net/ipv4/conf/" + intf + "/rp_filter"
		oshi.cmd(cmd)

def configure_env_ctrl(ctrl):
	global next_ctrl
	print "*** Configuring Environment For Controller", ctrl.name
	shutil.rmtree("/tmp/" + ctrl.name, ignore_errors=True)
	os.mkdir("/tmp/" + ctrl.name)
	configure_quagga(ctrl)
	ctrl.cmd("echo 1 > /proc/sys/net/ipv4/ip_forward ")
	ctrl.cmd("echo 0 > /proc/sys/net/ipv4/conf/all/rp_filter")
	for intf in ctrl.nameToIntf:
		cmd = "echo 0 > /proc/sys/net/ipv4/conf/" + intf + "/rp_filter"
		ctrl.cmd(cmd)

def configure_node(node):
	print "*** Configuring", node.name
	strip_ip(node)
	for net in nets:
		intfs_to_conf = net.belong(node.name)
		if(len(intfs_to_conf) > 0):
			for intf_to_conf in intfs_to_conf:
				sdn = False
				for tunnel in tunnels:
					if tunnel.belong(intf_to_conf) != None:
						sdn = True
						break
				if sdn == False:
					ip = net.give_me_next_ip()
					gw_ip = "%s.%s.%s.%s" % (net.subnet[0], net.subnet[1], net.subnet[2], 1)
					intf = intf_to_conf
					node.cmd('ip addr add %s/%s brd + dev %s' %(ip, ip_netbit, intf))
					node.cmd('ip link set %s up' % intf)
					node.cmd('route add default gw %s %s' %(gw_ip, intf))
				else:
					ip = tunnel.give_me_next_ip()
					intf = intf_to_conf
					node.cmd('ip addr add %s/%s brd + dev %s' %(ip, sdn_netbit, intf))
					node.cmd('ip link set %s up' % intf)

def configure_ovs(oshi, ctrl_ip, ctrl_port):
	print "*** Configuring OVS For", oshi.name
	path_ovs = "/tmp/" + oshi.name + "/ovs"
	os.mkdir(path_ovs)
	oshi.cmd("ovsdb-tool create " + path_ovs + "/conf.db")
	oshi.cmd("ovsdb-server " + path_ovs + "/conf.db --remote=punix:" + path_ovs + "/db.sock --remote=db:Open_vSwitch,manager_options" +
	" --no-chdir --unixctl=" + path_ovs + "/ovsdb-server.sock --detach")
	oshi.cmd("ovs-vsctl --db=unix:" + path_ovs + "/db.sock --no-wait init")
	oshi.cmd("ovs-vswitchd unix:" + path_ovs + "/db.sock -vinfo --log-file=" + path_ovs + "/ovs-vswitchd.log --no-chdir --detach")
	oshi.cmd("ovs-vsctl --db=unix:" + path_ovs + "/db.sock --no-wait add-br br-" + oshi.name)

	# Talking with controller
	oshi.cmd("ovs-vsctl --db=unix:" + path_ovs + "/db.sock --no-wait set-fail-mode br-" + oshi.name + " secure")
	oshi.cmd("ovs-vsctl --db=unix:" + path_ovs + "/db.sock --no-wait set-controller br-" + oshi.name + " tcp:%s:%s" %(ctrl_ip, ctrl_port))
	oshi.cmd("ovs-vsctl --db=unix:" + path_ovs + "/db.sock --no-wait set controller br-" + oshi.name + " connection-mode=out-of-band")

	# Setting DPID
	oshi.cmd("ovs-vsctl --db=unix:" + path_ovs + "/db.sock --no-wait -- set Bridge br-" + oshi.name + " other_config:datapath-id=" + oshi.dpid )

	eth_ports = []
	vi_ports = []
	n_ports = 1
	for intf in oshi.nameToIntf:
		if 'lo' not in intf:
			oshi.cmd("ovs-vsctl --db=unix:" + path_ovs + "/db.sock --no-wait add-port br-" + oshi.name + " " + intf)
			eth_ports.append(n_ports)
			n_ports = n_ports + 1
			viname = "vi%s" % strip_number(intf)
			oshi.cmd("ovs-vsctl --db=unix:" + path_ovs + "/db.sock --no-wait add-port br-" + oshi.name + " " + viname
			+ " -- set Interface " + viname + " type=internal")
			vi_ports.append(n_ports)
			n_ports = n_ports + 1
	if CORE_APPROACH == 'A':
		conf_flows_vlan_approach(oshi, eth_ports, vi_ports)
	elif CORE_APPROACH =='B':
		conf_flow_no_vlan_approach(oshi, eth_ports, vi_ports)

	if 'aos' in oshi.name:
		for i in range(0, len(L2nets)):
			intfs = L2nets[i].belong(oshi.name)
			if len(intfs) > 2 :
				print "Error More Than One AOSHI Interface In The Same Subnet"
				sys.exit(-2)
			if len(intfs) == 1 :
				print "*** Configuring Ingress/Egress Rules For %s In Network %s" % (oshi, L2nets[i].name)
				if L2nets[i].classification == 'A':
					conf_flows_ingress_egress_vlan_approach(oshi, i, intfs[0])
				elif L2nets[i].classification == 'B':
					conf_flows_ingress_egress_no_vlan_approach(oshi, i, intfs[0])

def conf_flows_ingress_egress_vlan_approach(oshi, i, intf):
	if CORE_APPROACH == 'A':
		print "*** Already Done Same Approach Between Core And Access"
	elif CORE_APPROACH == 'B':
		print "*** Add Rules For Vlan Access Approach"
		VLAN_IP = L2nets[i].VlanIP
		eth_intf = intf
		eth_port_number = convert_port_name_to_number(oshi, eth_intf)
		vi_intf = "vi%s" % strip_number(eth_intf)
		vi_port_number = convert_port_name_to_number(oshi, vi_intf)
		oshi.cmd("ovs-ofctl add-flow br-%s \"table=0,hard_timeout=0,priority=300,in_port=%s,dl_vlan=%s,actions=strip_vlan,resubmit(,1)\"" % (oshi.name, eth_port_number,VLAN_IP))
		oshi.cmd("ovs-ofctl add-flow br-%s \"table=1,hard_timeout=0,priority=300,in_port=%s,actions=mod_vlan_vid:%s,output:%s\"" % (oshi.name,vi_port_number,VLAN_IP,eth_port_number))

def conf_flows_ingress_egress_no_vlan_approach(oshi, i, intf):
	if CORE_APPROACH == 'B':
		print "*** Already Done Same Approach Between Core And Access"
	elif CORE_APPROACH == 'A':
		print "*** Add Rule For No Vlan Access Approach"
		VLAN_IP = 1 # Core Vlan
		eth_intf = intf
		eth_port_number = convert_port_name_to_number(oshi, eth_intf)
		vi_intf = "vi%s" % strip_number(eth_intf)
		vi_port_number = convert_port_name_to_number(oshi, vi_intf)
		oshi.cmd("ovs-ofctl del-flows br-%s in_port=%s,dl_vlan=%s" % (oshi.name,eth_port_number,VLAN_IP))
		oshi.cmd("ovs-ofctl add-flow br-%s hard_timeout=0,priority=300,in_port=%s,dl_vlan=%s,actions=mod_vlan_vid:%s,output:%s" % (oshi.name,eth_port_number,"0xffff",VLAN_IP,vi_port_number))
		oshi.cmd("ovs-ofctl add-flow br-%s hard_timeout=0,priority=300,in_port=%s,dl_vlan=%s,actions=strip_vlan,output:%s" % (oshi.name,vi_port_number,VLAN_IP,eth_port_number))

def conf_flows_vlan_approach(oshi, eth_ports, vi_ports):
	print "*** Configuring Flows Classifier A For", oshi
	VLAN_IP = 1
	size = len(eth_ports)
	i = 0
	for i in range(size):
		oshi.cmd("ovs-ofctl add-flow br-" + oshi.name + " hard_timeout=0,priority=300,in_port=" + str(eth_ports[i])
		+ ",dl_vlan=" + str(VLAN_IP) + ",action=output:" + str(vi_ports[i]))
		oshi.cmd("ovs-ofctl add-flow br-" + oshi.name + " hard_timeout=0,priority=300,in_port=" + str(vi_ports[i])
		+ ",dl_vlan=" + str(VLAN_IP) + ",action=output:" + str(eth_ports[i]))
	oshi.cmd("ovs-ofctl add-flow br-" + oshi.name + " hard_timeout=0,priority=400,dl_type=0x88cc,action=controller")
	oshi.cmd("ovs-ofctl add-flow br-" + oshi.name + " hard_timeout=0,priority=400,dl_type=0x8942,action=controller")

def conf_flow_no_vlan_approach(oshi, eth_ports, vi_ports):
	print "*** Configuring Flows Classifier B For", oshi
	size = len(eth_ports)
	i = 0
	oshi.cmd("ovs-ofctl add-flow br-" + oshi.name + " \"table=0,hard_timeout=0,priority=300,dl_vlan=0xffff,actions=resubmit(,1)\"")
	for i in range(size):
		oshi.cmd("ovs-ofctl add-flow br-" + oshi.name + " \"table=1,hard_timeout=0,priority=300,in_port=" + str(eth_ports[i])
		+ ",action=output:" + str(vi_ports[i]) + "\"")
		oshi.cmd("ovs-ofctl add-flow br-" + oshi.name + " \"table=1,hard_timeout=0,priority=300,in_port=" + str(vi_ports[i])
		+ ",action=output:" + str(eth_ports[i]) + "\"")
	oshi.cmd("ovs-ofctl add-flow br-" + oshi.name + " \"table=1,hard_timeout=0,priority=400,dl_type=0x88cc,action=controller\"")
	oshi.cmd("ovs-ofctl add-flow br-" + oshi.name + " \"table=1,hard_timeout=0,priority=400,dl_type=0x8942,action=controller\"")

def clean_env(oshi):
	print "*** Cleaning Environment For", oshi.name
	shutil.rmtree("/tmp/" + oshi.name, ignore_errors=True)

def configure_quagga(oshi):
	print "*** Configuring Quagga For", oshi.name
	path_quagga = "/tmp/" + oshi.name + "/quagga"
	os.mkdir(path_quagga)
	zebra_conf = open(path_quagga + "/zebra.conf","w")
	ospfd_conf = open(path_quagga + "/ospfd.conf","w")
	ospfd_nets = []
	ospfd_conf.write("hostname %s\n" % oshi.name)
	ospfd_conf.write("password zebra\n")
	ospfd_conf.write("log file /var/log/quagga/ospfd.log\n\n")
	zebra_conf.write("hostname %s\n" % oshi.name)
	zebra_conf.write("password zebra\n")
	zebra_conf.write("enable password zebra\n")
	zebra_conf.write("log file /var/log/quagga/zebra.log\n\n")
	for net in nets:
		intfs_to_conf = net.belong(oshi.name)
		if(len(intfs_to_conf) > 0):
			ospfd_nets.append(("%s.%s.%s.%s" %(net.subnet[0], net.subnet[1], net.subnet[2], 0), ip_netbit,(net.area)))
			for intf_to_conf in intfs_to_conf:
				ip = net.give_me_next_ip()
				if CORE_APPROACH == "A":
					intfname = configure_ospf_vlan_approach(oshi, intf_to_conf)
				elif CORE_APPROACH == "B":
					intfname = configure_ospf_no_vlan_approach(oshi, intf_to_conf)
				ospfd_conf.write("interface " + intfname + "\n")
				ospfd_conf.write("ospf cost %s\n" % net.cost)
				ospfd_conf.write("ospf hello-interval %s\n\n" % net.hello_int)
				zebra_conf.write("interface " + intfname + "\n")
				zebra_conf.write("ip address %s/%s\n" %(ip, ip_netbit))
				zebra_conf.write("link-detect\n\n")
	intfname = 'lo'
	if type(oshi) is RemoteController:
		ip = give_me_next_loopback()
	else:
		ip = oshi.loopback
	ospfd_conf.write("interface " + intfname + "\n")
	ospfd_conf.write("ospf cost %s\n" % 1)
	ospfd_conf.write("ospf hello-interval %s\n\n" % 2)
	zebra_conf.write("interface " + intfname + "\n")
	zebra_conf.write("ip address %s/%s\n" %(ip, 32))
	zebra_conf.write("link-detect\n\n")
	ospfd_conf.write("router ospf\n")
	ospfd_nets.append((ip, 32, "0.0.0.0"))
	for ospfd_net in ospfd_nets:
		ospfd_conf.write("network %s/%s area %s\n" %(ospfd_net[0], ospfd_net[1] , ospfd_net[2]))
	ospfd_conf.close()
	zebra_conf.close()
	oshi.cmd("chmod -R 777 /var/log/quagga")
	oshi.cmd("chmod -R 777 /var/run/quagga")
	oshi.cmd("chmod -R 777 %s" %(path_quagga))

def configure_ospf_vlan_approach(oshi, intfname):
	VLAN_IP = 1
	if 'c1' not in oshi.name:
		intfname = "vi%s" % (strip_number(intfname))
	oshi.cmd('ip link set %s up' % intfname)
	oshi.cmd('modprobe 8021q')
	oshi.cmd('vconfig add %s %s' % (intfname, VLAN_IP))
	intfname = intfname + "." + str(VLAN_IP)
	return intfname

def configure_ospf_no_vlan_approach(oshi, intfname):
	if 'c1' not in oshi.name:
		intfname = "vi%s" % (strip_number(intfname))
	oshi.cmd('ip link set %s up' % intfname)
	return intfname

def configure_vll_pusher(net):
	print "*** Create Configuration File For Vll Pusher"
	path = vll_path + "vll_pusher.cfg"
	vll_pusher_cfg = open(path,"w")
	for i in range(0, len(LHS_tunnel_aoshi)):
		aoshi = LHS_tunnel_aoshi[i]
		lhs_dpid = net.getNodeByName(aoshi).dpid
		lhs_dpid = ':'.join(s.encode('hex') for s in lhs_dpid.decode('hex'))
		port = LHS_tunnel_port[i]
		lhs_port = port
		aoshi = RHS_tunnel_aoshi[i]
		rhs_dpid = net.getNodeByName(aoshi).dpid
		rhs_dpid = ':'.join(s.encode('hex') for s in rhs_dpid.decode('hex'))
		port = RHS_tunnel_port[i]
		rhs_port = port
		vll_pusher_cfg.write("%s|%s|%s|%s|%d|%d|\n" % (lhs_dpid, rhs_dpid, lhs_port, rhs_port, LHS_tunnel_vlan[i], RHS_tunnel_vlan[i]))
	vll_pusher_cfg.close()
	root = Node( 'root', inNamespace=False )
	root.cmd("chmod 777 %s" %(path))

def init_net(net):
	"Init Function"
	root = Node( 'root', inNamespace=False )
	root.cmd('stop avahi-daemon')
	root.cmd('killall dhclient')
	root.cmd('killall zebra')
	root.cmd('killall ospfd')
	fixEnvironment()
	print "*** Restarting Network Manager"
	time.sleep(10)
	root.cmd('service network-manager restart')
	time.sleep(2)
	net.start()

	# Configure the ctrl
	for ctrl in ctrls:
		configure_env_ctrl(ctrl)
	for oshi in oshis:
		configure_env_oshi(oshi)
	for oshi in oshis:
		strip_ip(oshi)
		path_quagga_conf = "/tmp/" + oshi.name + "/quagga"
		oshi.cmd("%szebra -f %s/zebra.conf -A 127.0.0.1 &" %(path_quagga_exec, path_quagga_conf))
		oshi.cmd("%sospfd -f %s/ospfd.conf -A 127.0.0.1 &" %(path_quagga_exec, path_quagga_conf))
	for aoshi in aoshis:
		configure_env_oshi(aoshi)
	for aoshi in aoshis:
		strip_ip(aoshi)
		path_quagga_conf = "/tmp/" + aoshi.name + "/quagga"
		aoshi.cmd("%szebra -f %s/zebra.conf -A 127.0.0.1 &" %(path_quagga_exec, path_quagga_conf))
		aoshi.cmd("%sospfd -f %s/ospfd.conf -A 127.0.0.1 &" %(path_quagga_exec, path_quagga_conf))
	for ctrl in ctrls:
		path_quagga_conf = "/tmp/" + ctrl.name + "/quagga"
		ctrl.cmd("%szebra -f %s/zebra.conf -A 127.0.0.1 &" %(path_quagga_exec, path_quagga_conf))
		ctrl.cmd("%sospfd -f %s/ospfd.conf -A 127.0.0.1 &" %(path_quagga_exec, path_quagga_conf))
	print "*** Configuring Hosts"
	i = 0
	for i in range(len(hosts)):
		host = net.getNodeByName(hosts[i])
		configure_node(host)
	configure_standalone_sw(switches)
	configure_l2_accessnetwork()
	# Configure VLL Pusher
	configure_vll_pusher(net)
	print "*** Type 'exit' or control-D to shut down network"
	CLI( net )
	net.stop()
	subprocess.call(["sudo", "mn", "-c"], stdout=None, stderr=None)
	for oshi in oshis:
		clean_env(oshi)
	for aoshi in aoshis:
		clean_env(aoshi)
	for ctrl in ctrls:
		clean_env(ctrl)
	path = vll_path + "vlls.json"
	if(os.path.exists(path)):
		print "*** Remove Vlls DB File"
		os.remove(path)
	print '*** Unmounting host bind mounts'
	root.cmd('service network-manager restart')
	root.cmd('start avahi-daemon')
	root.cmd('killall ovsdb-server')
	root.cmd('killall ovs-vswitchd')
	root.cmd('killall zebra')
	root.cmd('killall ospfd')
	root.cmd('/etc/init.d/openvswitch-switch restart')
	unmountAll()

def parse_cmd_line():
	parser = argparse.ArgumentParser(description='Mininet Deployer')
	parser.add_argument('--topology', dest='topoInfo', action='store', default='mesh:3', help='Topology Info topo:param, e.g., mesh:3 or file:topo.json')
	args = parser.parse_args()
	if len(sys.argv)==1:
    		parser.print_help()
    		sys.exit(1)
	data = args.topoInfo.split(":")
	return (data[0], data[1])

def check_precond():
	unmountAll()
	if vll_path == "" or path_quagga_exec == "":
		print "Error Set Environment Variable At The Beginning Of File"
		sys.exit(-2)

if __name__ == '__main__':

	net = None
	lg.setLogLevel('info')
	(topo, param) = parse_cmd_line()
	check_precond()
	if topo == 'file':
		print "*** Create Topology From File:", param
		net = buildTopoFromFile(param)
	elif topo == 'mesh':
		print "*** Create Built-in Topology mesh[%s]" % param
		net = Mesh(int(param))
	else:
		print "*** Create Topology From Networkx:", topo, param
		net = buildTopoFromNx(topo,param)
	init_net(net)

max_stars_repo_name: netgroup/Dreamer-Mininet-Deployer

max_issues_repo_name: netgroup/Dreamer-Mininet-Deployer

max_forks_repo_name: netgroup/Dreamer-Mininet-Deployer

reason: import numpy,import networkx


text:
# # basicMLpy.nearest_neighbors module
import numpy as np
from scipy import linalg
from .utils import euclidean_distance
def get_neighbors(x, row_num:int, n_neighbors:int):
    """
    Gets the K nearest neighbors relative to a point in the dataset.
    Inputs:
        x: array
            input the array of input points.
        row_num: int
            input the index(row number) of the desired point to calculate the k nearest neighbors.
        n_neighbors: int
            input the number of neighbors to be calculated.
    Returns:
        neighbors: array
            outputs the array of the k nearest neighbors to the inputed point.
    """
    distances = list()
    for row in x:
        dist = euclidean_distance(x[row_num,:], row)
        distances.append((row, dist))
    distances.sort(key=lambda tup: tup[1])
    neighbors = list()
    for i in range(n_neighbors):
        neighbors.append(distances[i][0])
    return np.array(neighbors)
def make_prediction(x,row_num:int,n_neighbors:int,weights:str):
    """
    Predicts the output value based on the k nearest neighbors relative to an input point.
    Inputs:
        x: array
            input the array of input points.
        row_num: int
            input the index(row number) of the desired point to make the prediction.
        n_neighbors: int
            input the number of neighbors to be calculated.
        weights: string
            input the type of weight to be used in the calculation. weights can be: 'uniform'(uniform weights,i.e, all points on the neighborhood are weighted equally) or 'distance'(weight points by the inverse of their distance).
    Returns:
        predictions: float
            outputs the prediction calculated based on the input point(row_num).

    """
    neighbors = get_neighbors(x,row_num,n_neighbors)
    outputs = [points[neighbors.shape[1] - 1] for points in neighbors]
    if weights == 'uniform':
        predictions = np.round(sum(outputs)/n_neighbors)
        return predictions
    elif weights == 'distance':
        for i in range(len(outputs)):
            outputs[i] = outputs[i]/(i+1)
        predictions = np.round(sum(outputs)/n_neighbors)
        return predictions
    else:
        raise ValueError("Please insert a valid weight")
class NearestNeighbors:
    """
    Class of the K-Nearest Neighbors algorithm.
    Fits both regression and classification problems.
    This algorithm performs a brute force search, meaning that it performs poorly on large datasets, since it scales according to O[DN^2], where N is the number of samples and D is the number of dimensions.
    Methods:
        predict(x,y) -> Predict value for X.
        kneighbors(row_num,n_neighbors) -> Gets the k-nearest neighbors from x.
    """
    def __init__(self,n_neighbors = 5,weights = 'uniform'):
        """
        Initial parameters.
        Inputs:
            n_neighbors: int
                input the number of neighbors to be calculated; default is set to 5.
            weights: string
                input the type of weight to be used in the calculation. weights can be: 'uniform'(uniform weights,i.e, all points on the neighborhood are weighted equally) or 'distance'(weight points by the inverse of their distance); default is set to 'uniform'.
        """
        self.n_neighbors = n_neighbors
        self.weights = weights
    def predict(self,x,y):
        """
        Makes a prediction based on a given dataset.
        Inputs:
            x: array
                input the array of input points.
            y: array
                input the array of output points.
        Returns:
            predictions: array
                outputs the array of predictions.
        """
        self.input = x
        self.output = y.reshape((-1,1))
        self.data = np.hstack((self.input,self.output))
        predictions = np.zeros((len(self.output)))
        for i in range(len(predictions)):
            predictions[i] = make_prediction(self.data,i,self.n_neighbors,self.weights)
        return predictions
    def kneighbors(self,row_num:int,n_neighbors:int):
        """
        Gets the k-nearest neighbors to a certain point.
        Inputs:
            row_num: int
                input the index(row number) of the desired point to calculate the k nearest neighbors.
            n_neighbors: int
                input the number of neighbors to be calculated.
        """
        kneighbors = get_neighbors(self.input,row_num,n_neighbors)
        return kneighbors

ext: py

max_stars_repo_name: HenrySilvaCS/basicMLpy

max_issues_repo_name: HenrySilvaCS/basicMLpy

max_forks_repo_name: HenrySilvaCS/basicMLpy

reason: import numpy,from scipy


text:
#!/usr/bin/env python

import os
import argparse
import numpy as np
import pandas as pd
from collections import namedtuple
from scipy import optimize
import matplotlib.pyplot as plt

import seaborn

def main():
    """Fit pH and cl titrations where data are spectra.
    input: spectra_table.csv; _note
    output: pK; spK; and plot (*.pdf)
    """
    description = "Fit spectra from a pH or a Cl titration"
    parser = argparse.ArgumentParser(description=description)
    parser.add_argument('csvtable',
                        help="a table containing spectra arranged in columns")
    parser.add_argument('note_file',
                        help="a file to describe the titration")
    parser.add_argument("-d", '--out', dest='out',
                        help="destination directory (default: .)")
    parser.add_argument("-m", "--method-of-analysis", action="store",
                        default="svd", choices=["svd", "band"],
                        dest='analysis_method')
    parser.add_argument("-t", "--titration-of", action="store", default="pH",
                        choices=["pH", "cl"], dest='titration_type')
    parser.add_argument("-b", "--band-interval", action="store", nargs=2,
                        type=int, dest='band',
                        help='Integration interval from <1> to <2>')
    parser.add_argument("-v", "--verbose", action="store_true",
                        help="increase output verbosity")
    args = parser.parse_args()
    # input of spectra (csv) and titration (note) data
    csv = pd.read_csv(args.csvtable)
    note_file = pd.read_table(args.note_file)
    note_file = note_file[note_file['mutant'] != 'buffer']
    # TODO aggregation logic for some pH or cloride
    Note = namedtuple("note", "wells conc")
    # pH or Cl titration
    if args.titration_type == "cl":
        note = Note(list(note_file.well), list(note_file.Cl))

        def fz(Kd, p, x):
            return ((p[0] + p[1] * x / Kd) / (1 + x / Kd))
    elif args.titration_type == "pH":
        note = Note(list(note_file.well), list(note_file.pH))

        def fz(pK, p, x):
            return (p[1] + p[0] * 10 ** (pK - x)) / (1 + 10 ** (pK - x))
    df = csv[note.wells]
    df.index = csv['lambda']
    conc = np.array(note.conc)
    # sideeffect print input data
    if args.verbose:
        print(csv)
        print(note)
        print('conc vector\n', conc)
        print('DataFrame\n', df)
    if args.analysis_method == 'svd':
        # svd on difference spectra
        ddf = df.sub(df.iloc[:, 0], axis=0)
        u, s, v = np.linalg.svd(ddf)
        # fitting
        result = fit_titration(fz, conc, v[0, :])
        # plotting
        seaborn.set_style('ticks')
        fig1 = plt.figure(figsize=(12, 8))
        ax1 = fig1.add_axes([0.05, 0.65, 0.32, 0.31])
        plt.grid(True)
        ax2 = fig1.add_axes([0.42, 0.65, 0.32, 0.31])
        plt.grid(True)
        ax1.plot(df)
        ax2.plot(ddf.index, u[:, 0], 'k-', lw=3)
        ax2.plot(ddf.index, u[:, 1], 'b--')
        ax3 = fig1.add_axes([0.80, 0.65, 0.18, 0.31], yscale='log',
                            xticks=[1, 2, 3, 4], title='autovalues')
        ax3.bar([1, 2, 3, 4], (s**2 / sum(s**2))[:4],
                align='center', alpha=0.7, width=0.66)
        ax4 = fig1.add_axes([0.05, 0.08, 0.50, 0.50], title="fitting")
        ax5 = fig1.add_axes([0.63, 0.08, 0.35, 0.50],
                            title='SVD coefficients',
                            xlabel='1$^{st}$ autovector',
                            ylabel='2$^{nd}$ autovector')
        ax4.scatter(conc, v[0, :])
        xmin = conc.min()
        xmax = conc.max()
        xmax += (xmax - xmin) / 7
        xlin = np.linspace(xmin, xmax, 100)
        ax4.plot(xlin, fz(result.K, [result.SA, result.SB], xlin))
        result.s1 = str(round(result.K, 2)) + ' \u00B1 '
        result.s1 = result.s1 + str(round(result.sK, 2))
        plt.figtext(.26, .54, result.s1, size=20)
        ax5.plot(v[:, 1], v[:, 2], lw=0.8)
        for x, y, l in zip(v[:, 1], v[:, 2], note.wells):
            ax5.text(x, y, l)
        if 1 == 1:
            kd = []
            sa = []
            sb = []
            for i in range(100):
                boot_idxs = np.random.randint(0, len(ddf.columns) - 1,
                                                      len(ddf.columns))
                ddf2 = df.iloc[:, boot_idxs]
                conc2 = conc[boot_idxs]
                u, s, v = np.linalg.svd(ddf2)
                result2 = fit_titration(fz, conc2, v[0, :])
                kd.append(result2.K)
                sa.append(result2.SA)
                sb.append(result2.SB)
            bs = pd.DataFrame({'kd': kd, 'SA': sa, 'SB': sb})
            bs.to_csv('bs.txt')
    elif args.analysis_method == 'band':
        # fitting
        try:
            ini = args.band[0]
            fin = args.band[1]
            y = []
            for c in df.columns:
                y.append(df[c].loc[ini: fin].sum())
            result = fit_titration(fz, conc, y)
        except:
            print('''bands [{0}, {1}] not in index.
                  Try other values'''.format(ini, fin))
            raise
        # plotting
        fig1 = plt.figure(figsize=(12, 8))
        ax4 = fig1.add_axes([0.05, 0.08, 0.50, 0.50], title="fitting")
        ax4.scatter(conc, y)
        xmin = conc.min()
        xmax = conc.max()
        xmax += (xmax - xmin) / 7
        xlin = np.linspace(xmin, xmax, 100)
        ax4.plot(xlin, fz(result.K, [result.SA, result.SB], xlin))
        result.s1 = str(round(result.K, 2)) + ' \u00B1 '
        result.s1 = result.s1 + str(round(result.sK, 2)) + '[' + str(ini)
        result.s1 = result.s1 + ':' + str(fin) + ']'
        plt.figtext(.26, .54, result.s1, size=20)
        for x, y, l in zip(conc, y, note.wells):
            ax4.text(x, y, l)

    # output
    f_csv_shortname = os.path.splitext(os.path.split(args.csvtable)[1])[0]
    f_note_shortname = os.path.splitext(os.path.split(args.note_file)[1])[0]
    f_out = f_csv_shortname.join([args.analysis_method + "_", "_"])
    f_out = f_out + f_note_shortname
    if args.out:
        if not os.path.isdir(args.out):
            os.makedirs(args.out)
        f_out = os.path.join(args.out, f_out)
    fig1.savefig(f_out + ".pdf")

    print("best-fitting using: ", args.analysis_method)
    print("spectra csv file: ", f_csv_shortname)
    print("note file: ", f_note_shortname)
    print("K = ", round(result.K, 3))
    print("sK = ", round(result.sK, 3))
    print("SA = ", round(result.SA, 3))
    print("sSA = ", round(result.sSA, 3))
    print("SB = ", round(result.SB, 3))
    print("sSB = ", round(result.sSB, 3))

def fit_titration(fz, x, y):
    """Fit a dataset (x, y) using a single-site binding model provided by the
    function **fz** that defines a constant *K* and 2 plateau *SA* and *SB*.

    Parameters
    ----------

    x, y : numpy arrays

    Return
    ------

    res : namedtuple
        contains the least square results

    """
    def ssq(p, x, y1):
        return np.r_[y1 - fz(p[0], p[1:3], x)]
    # plateau calculation
    df = pd.DataFrame({'x': x, 'y': y})
    SA = df.y[df.x == min(df.x)].values[0]
    SB = df.y[df.x == max(df.x)].values[0]
    p0 = np.r_[7.1, SA, SB]
    p, cov, info, msg, success = optimize.leastsq(ssq, p0, args=(x, y),
                                                  full_output=True, xtol=1e-11)
    # ftol=1.49012e-23,
    res = namedtuple("Result", "success msg df chisqr K sK SA sSA SB sSB")
    res.msg = msg
    res.success = success
    if 1 <= success <= 4:
        chisq = sum(info['fvec'] * info['fvec'])
        res.df = len(y) - len(p)
        res.chisqr = chisq / res.df
        res.K = p[0]
        res.sK = np.sqrt(cov[0][0] * res.chisqr)
        res.SA = p[1]
        res.sSA = np.sqrt(cov[1][1] * res.chisqr)
        res.SB = p[2]
        res.sSB = np.sqrt(cov[2][2] * res.chisqr)
    return res

main()

max_stars_repo_name: darosio/ClopHfit

max_issues_repo_name: darosio/ClopHfit

max_forks_repo_name: darosio/ClopHfit


text:
"""
Error Profiling Investigations
"""

from fairtest.investigation import Investigation, metric_from_string
from fairtest.modules.metrics import NMI, CondDIFF, CORR, CondNMI, CondCORR
import numpy as np
import logging
from sklearn.preprocessing import LabelEncoder

class ErrorProfiling(Investigation):
    """
    A FairTest Error Profiling Investigation
    """

    def __init__(self, data_source, protected, output, ground_truth,
                 expl=None, metrics=None, random_state=None, to_drop=None):
        """
        Initializes a FairTest Testing Investigation.

        Parameters
        ----------
        data_source :
            the data source
        protected :
            list of names of protected features
        output :
            name of output feature
        ground_truth :
            name of ground truth column in the dataset
        expl :
            name of explanatory feature
        metrics :
            dictionary of custom metrics indexed by a protected feature
        random_state :
            seed for random generators
        to_drop :
            features to drop from the training set
        """

        logging.info('New Error Profiling Investigation')

        data_source = data_source.duplicate()
        train_data = data_source.train_data

        if len(np.asarray([output]).flatten()) != 1:
            raise ValueError('Error Profiling investigation excepts a single '
                             'target')

        if output not in train_data.columns:
            raise ValueError('Unknown target feature %s' % output)

        if len(np.asarray([ground_truth]).flatten()) != 1:
            raise ValueError('Error Profiling investigation excepts a single '
                             'ground truth')

        if ground_truth not in train_data.columns:
            raise ValueError('Unknown ground truth feature %s' % ground_truth)

        self.original_output = output
        self.ground_truth = ground_truth
        self.error_name = None

        train_data, error_name = self.compute_errors(train_data)
        self.error_name = error_name

        if train_data.dtypes[error_name] == np.object:
            data_source.encoders[error_name] = LabelEncoder()
            train_data[error_name] = data_source.encoders[error_name].\
                fit_transform(train_data[error_name])
            logging.info('Encoding Feature %s' % error_name)

        data_source.train_data = train_data
        Investigation.__init__(self, data_source, protected, error_name,
                               expl, metrics, random_state, to_drop)

    def preprocess_test_data(self, data):
        """
        Pre-processing stage for the testing data. Computes the appropriate
        error metric for the test data.

        Parameters
        ----------
        data :
            the test data
        """
        data, _ = self.compute_errors(data)

        if self.error_name in self.encoders:
            data[self.error_name] = self.encoders[self.error_name].\
                transform(data[self.error_name])

        return data

    def compute_errors(self, data):
        """
        Computes an error metric over the data

        Parameters
        ----------
        data :
            the data
        """
        output = self.original_output
        ground_truth = self.ground_truth

        if self.error_name == "Bin Class. Error" or \
                (set(data[output].unique()) == set([0, 1]) and
                 set(data[ground_truth].unique()) == set([0, 1])):

            # binary classification
            logging.info('Computing Binary Classification Error')
            error_name = "Bin Class. Error"
            data[error_name] = \
                ['Correct' if pred == truth else 'FP' if pred else 'FN'
                 for (pred, truth) in zip(data[output],
                                          data[ground_truth])]
        elif self.error_name == "Class. Error" or \
                        data.dtypes[output] == np.object:

            # multi-valued classification
            logging.info('Computing Multivalued Classification Error')
            error_name = "Class. Error"
            data[error_name] = \
                ['Correct' if pred == truth else 'Incorrect' for (pred, truth)
                 in (zip(data[output], data[ground_truth]))]
        else:
            # regression
            logging.info('Computing Absolute Regression Error')
            error_name = "Abs. Error"
            data[error_name] = abs(np.array(data[output]) -
                                   np.array(data[ground_truth]))

        data = data.drop(ground_truth, axis=1)
        data = data.drop(output, axis=1)

        return data, error_name

    def set_default_metrics(self):
        out = self.output
        expl = self.feature_info[self.expl] if self.expl else None

        if out.num_labels != 1:
            raise ValueError('Error Profiling investigation excepts a single '
                             'target')

        for sens_str in self.sens_features:
            sens = self.feature_info[sens_str]

            if sens_str in self.metrics:
                if isinstance(self.metrics[sens_str], basestring):
                    self.metrics[sens_str] = \
                        metric_from_string(self.metrics[sens_str])

                self.metrics[sens_str].validate(sens, out, expl)
            else:
                if expl is not None:
                    if not expl.arity:
                        raise ValueError(
                            'Only categorical explanatory features allowed')
                    if sens.arity == 2 and out.arity == 2:
                        logging.info('Choosing metric CondDIFF for feature %s' %
                                     sens_str)
                        self.metrics[sens_str] = CondDIFF()
                    elif sens.arity and out.arity:
                        logging.info('Choosing metric CondNMI for feature %s'
                                     % sens_str)
                        self.metrics[sens_str] = CondNMI()
                    elif not (sens.arity > 2 or out.arity > 2):
                        logging.info('Choosing metric CondCORR for feature %s'
                                     % sens_str)
                        self.metrics[sens_str] = CondCORR()
                    else:
                        raise ValueError('Only categorical protected features '
                                         'and outputs supported')
                elif sens.arity and out.arity:
                    logging.info('Choosing metric NMI for feature %s'
                                 % sens_str)
                    self.metrics[sens_str] = NMI()
                else:
                    if sens.arity > 2 or out.arity > 2:
                        raise ValueError('No Metric available for continuous '
                                         'and multi-valued features')
                    logging.info('Choosing metric CORR for feature %s' %
                                 sens_str)
                    self.metrics[sens_str] = CORR()

max_stars_repo_name: columbia/fairtest

max_issues_repo_name: columbia/fairtest

max_forks_repo_name: columbia/fairtest

reason: import numpy


text:
# MAIN PROGRAM
# AUTHOR: Khurrum Nasim

import matplotlib.pyplot as plt
import glob
import time
import collections
from sklearn.svm import LinearSVC
from sklearn.preprocessing import StandardScaler
from scipy.ndimage.measurements import label
from sklearn.cross_validation import train_test_split
from moviepy.editor import VideoFileClip
from function import *

cars = glob.glob('vehicles/vehicles/*/*.png')
notcars = glob.glob('temp-non-vehicles/non-vehicles/*/*.png')

color_space = 'YCrCb' # Can be RGB, HSV, LUV, HLS, YUV, YCrCb
orient = 9  # HOG orientations
pix_per_cell = 8 # HOG pixels per cell
cell_per_block = 2 # HOG cells per block
hog_channel = "ALL" # Can be 0, 1, 2, or "ALL"
spatial_size = (32, 32) # Spatial binning dimensions
hist_bins = 32    # Number of histogram bins
spatial_feat = True # Spatial features on or off
hist_feat = True # Histogram features on or off
hog_feat = True # HOG features on or off
y_start_stop = [400, 720] # Min and max in y to search in slide_window()
x_start_stop = [400, 1280]

car_features = extract_features(cars, color_space=color_space,
                        spatial_size=spatial_size, hist_bins=hist_bins,
                        orient=orient, pix_per_cell=pix_per_cell,
                        cell_per_block=cell_per_block,
                        hog_channel=hog_channel, spatial_feat=spatial_feat,
                        hist_feat=hist_feat, hog_feat=hog_feat)
noncar_features = extract_features(notcars, color_space=color_space,
                                   spatial_size=spatial_size, hist_bins=hist_bins,
                                   orient=orient, pix_per_cell=pix_per_cell,
                                   cell_per_block=cell_per_block,
                                   hog_channel=hog_channel, spatial_feat=spatial_feat,
                                   hist_feat=hist_feat, hog_feat=hog_feat)

# create an array stack of feature vectors
X = np.vstack((car_features, noncar_features)).astype(np.float64)

# Define the labels vector
Y = np.hstack((np.ones(len(car_features)), np.zeros(len(noncar_features))))

# split up data into randomized training and test sets
rand_state = np.random.randint(0, 100)
X_train, X_test, Y_train, Y_test = train_test_split(
    X, Y, test_size=0.2, random_state=rand_state)

# Fit a per-column scaler
X_scaler = StandardScaler().fit(X_train)
# Apply the scaler to X
X_train = X_scaler.transform(X_train)
X_test = X_scaler.transform(X_test)

print('Using:',orient,'orientations',pix_per_cell,
    'pixels per cell and', cell_per_block,'cells per block')
print('Feature vector length:', len(X_train[0]))

# Use a linear SVC
svc = LinearSVC()
# check SVC training time
t=time.time()
svc.fit(X_train, Y_train)
t2 = time.time()
print(round(t2-t, 2), 'Seconds to train SVC...')
# Check the score of the SVC
print('Test Accuracy of SVC = ', round(svc.score(X_test, Y_test), 4))
# Check the prediction time for a single sample
t=time.time()

image = mpimg.imread('test_images/test4.jpg')
draw_image = np.copy(image)
img = image
heat_img = image

# Uncomment the following line if you extracted training
# data from .png images (scaled 0 to 1 by mpimg) and the
# image you are searching is a .jpg (scaled 0 to 255)
image = image.astype(np.float32)/255

windows = slide_window(image, x_start_stop=x_start_stop, y_start_stop=y_start_stop,
                    xy_window=(96, 96), xy_overlap=(0.5, 0.5))

hot_windows = search_windows(image, windows, svc, X_scaler, color_space=color_space,
                        spatial_size=spatial_size, hist_bins=hist_bins,
                        orient=orient, pix_per_cell=pix_per_cell,
                        cell_per_block=cell_per_block,
                        hog_channel=hog_channel, spatial_feat=spatial_feat,
                        hist_feat=hist_feat, hog_feat=hog_feat)

window_img = draw_boxes(draw_image, hot_windows, color=(0, 0, 255), thick=6)
plt.imshow(window_img)

ystart = 400
ystop = 656
scale = 1.5

out_img, bboxes = detect_cars(img, ystart, ystop, scale, svc, X_scaler, orient, pix_per_cell, cell_per_block,
                              spatial_size, hist_bins)
print("show 1")
plt.show()
#plt.imshow(out_img)

image = out_img
box_list = bboxes

heat = np.zeros_like(image[:,:,0]).astype(np.float)
# Add heat to each box in box list
heat = add_heat(heat, box_list)

# Apply threshold to help remove false positives
heat = apply_threshold(heat, 1)

# Visualize the heatmap when displaying
heatmap = np.clip(heat, 0, 255)

# Find final boxes from heatmap using label function
labels = label(heatmap)
draw_img = draw_labeled_bboxes(np.copy(heat_img), labels)

fig = plt.figure()
plt.subplot(121)
plt.imshow(draw_img)
plt.title('Car Positions')
plt.subplot(122)
plt.imshow(heatmap, cmap='hot')
plt.title('Heat Map')
plt.show()
fig.tight_layout()

heatmaps = collections.deque(maxlen=30)

def pipeline(image):
    global heats

    img = image
    heat = np.zeros_like(image[:, :, 0]).astype(np.float)
    out_img, bboxes = detect_cars(img, ystart, ystop, scale, svc, X_scaler, orient, pix_per_cell, cell_per_block,
                                  spatial_size, hist_bins)
    box_list = bboxes
    heat = add_heat(heat, box_list)
    heatmaps.append(heat)
    heatmap_sum = sum(heatmaps)
    # Apply threshold to help remove false positives

    heat = apply_threshold(heatmap_sum, 1)

    # Visualize the heatmap when displaying
    heatmap = np.clip(heat, 0, 255)

    # Find final boxes from heatmap using label function
    labels = label(heatmap_sum)
    draw_img = draw_labeled_bboxes(np.copy(image), labels)
    return draw_img

video = VideoFileClip("project_video.mp4")
clip = video.fl_image(pipeline) #NOTE: this function expects color images!!
output = "output.mp4"
clip.write_videofile(output, audio=False)

max_stars_repo_name: knasim/Vehicle-Detection-And-Tracking

max_issues_repo_name: knasim/Vehicle-Detection-And-Tracking

max_forks_repo_name: knasim/Vehicle-Detection-And-Tracking

reason: from scipy


text:
import os, sys
from time import time, sleep
import numpy as np
import matplotlib.pyplot as plt

from mesostat.metric.metric import MetricCalculator

def generate_data(nTrial, nData, isSelfPredictive, isLinear, isShifted):
    dataShape = (nTrial, nData)

    if isSelfPredictive:
        src = np.outer(np.ones(nTrial), np.linspace(0, 1, nData))
    else:
        src = np.random.normal(0, 1, dataShape)

    if isLinear:
        x = src
        y = 1 - 2 * src
    else:
        x = np.cos(2 * np.pi * src)
        y = np.sin(2 * np.pi * src)

    x += 0.2 * np.random.normal(0, 1, dataShape)
    y += 0.2 * np.random.normal(0, 1, dataShape)

    if isShifted:
        y = np.hstack(([y[-1]], y[:-1]))

    return np.array([x, y])

def write_phase_space_fig(data, path, dataName):
    plt.figure()
    plt.plot(data[0].flatten(), data[1].flatten(), '.')
    plt.savefig(os.path.join(path, "data_" + dataName + ".png"))
    plt.close()

def write_fc_lag_p_fig(rez, path, rezKey):
    # Plot results
    fig, ax = plt.subplots()
    ax.imshow(rez)

    print(rezKey)
    outNameBare = "_".join([k for k in rezKey if k is not None])
    plt.savefig(os.path.join(path, outNameBare + ".png"))
    plt.close()

########################
# Generic parameters
########################
outpath = "tmp_imgs"

nTime = 100
nTrial = 10
nCoreArr = np.arange(1,5)

########################
# Sweep parameters
########################

algDict = {
    "Metric"       : ["crosscorr", "crosscorr", "BivariateMI", "BivariateMI", "BivariateMI"],
    "Library"      : ["corr", "spr", None, None, None, None],
    "CMI"          : [None, None, "OpenCLKraskovCMI", "JidtGaussianCMI", "JidtKraskovCMI"],
}

excludeCMI = ["OpenCLKraskovCMI"]

timesDict = {}
ptests = []
for dataName in ["Linear", "Circular"]:
    data = generate_data(nTrial, nTime, False, dataName == "Linear", False)

    write_phase_space_fig(data, outpath, dataName)

    for metricName, estimator, cmi in zip(*algDict.values()):
        # Get label
        taskKey = (dataName, metricName, estimator, cmi)

        if cmi not in excludeCMI:
            print("computing", taskKey)

            # Get settings
            metricSettings = {
                'lag': 1,
                'min_lag_sources': 1,
                'max_lag_sources': 1,
                "cmi_estimator": cmi,
                'est': estimator,
                'parallelTrg': True
            }

            # Compute performance metric
            timesDict[taskKey] = []

            for nCore in nCoreArr:
                tStart = time()
                mc = MetricCalculator(nCore=nCore)
                mc.set_data(data, 'prs', zscoreDim="rs")
                rez = mc.metric3D(metricName, "", metricSettings=metricSettings)
                if metricName != "crosscorr":
                    rez = rez[0]

                timesDict[taskKey] += [time() - tStart]

                write_fc_lag_p_fig(rez, outpath, taskKey)

            print("Sleeping...")
            sleep(1)

print("Off diagonal p-values")
print(np.array(ptests))

plt.figure()
plt.title("Strong scaling")
for k,v in timesDict.items():
    plt.semilogy(nCoreArr, v, label=str(k))

plt.xlabel("nCores")
plt.ylabel("runtime, seconds")
plt.legend()
plt.show()

max_stars_repo_name: HelmchenLabSoftware/mesostat-dev

max_issues_repo_name: HelmchenLabSoftware/mesostat-dev

max_forks_repo_name: HelmchenLabSoftware/mesostat-dev


text:
"""https://www.kaggle.com/grfiv4/plot-a-confusion-matrix."""
from itertools import product
from typing import List, Union

import matplotlib.pyplot as plt
import numpy as np
from sklearn.metrics import confusion_matrix

from chitra.logging import logger

def detect_multilabel(labels: Union[List, np.ndarray]):
    if isinstance(labels, (np.ndarray, list, tuple)):
        labels = np.unique(labels)
    else:
        raise UserWarning(
            f"expect data type for label was list or np.ndarray but got {type(labels)}"
        )

    if len(labels) > 2:
        return True
    return False

def cm_accuracy(cm: np.ndarray):
    return np.trace(cm) / float(np.sum(cm))

def plot_confusion_matrix(
    y_pred: Union[np.ndarray, List],
    y_true: Union[np.ndarray, List],
    display_labels=None,
    include_values: bool = True,
    title: str = "Confusion Matrix",
    cmap: str = None,
):
    if detect_multilabel(y_true):
        logger.warning("You might want to use multi-label version!")

    if display_labels is None:
        display_labels = np.unique(y_true)

    n_classes = len(display_labels)
    tick_marks = np.arange(n_classes)

    if cmap is None:
        cmap = plt.get_cmap("Blues")

    cm = confusion_matrix(y_true, y_pred)
    accuracy = cm_accuracy(cm)
    error = 1 - accuracy

    plt.imshow(cm, cmap=cmap)

    if include_values:
        for i, j in product(range(n_classes), range(n_classes)):
            plt.text(i, j, "{:,}".format(cm[i, j]))

    plt.xticks(tick_marks, display_labels, rotation=45)
    plt.yticks(tick_marks, display_labels)
    plt.title(title)
    plt.xlabel(f"Predicted Label\nAccuracy={accuracy:0.4f}; Error={error:0.4f}")
    plt.ylabel("True Label")

    plt.show()

max_stars_repo_name: sukkritsharmaofficial/chitra

max_issues_repo_name: sukkritsharmaofficial/chitra

max_forks_repo_name: sukkritsharmaofficial/chitra


text:
#!/usr/local/PROGRAMS/XTAL/GIT-AMi_Image_Analysis/python/venvs/AMI_IMAGE_ANALYSIS_TENSORFLOW1/bin/python
# -*- coding: utf-8 -*-
"""
    This file was adpated from the following source:
    https://github.com/sjawhar/focus-stacking/tree/master/focus_stack

    This file is taken from
    https://github.com/dakota0064/Fluorescent_Robotic_Imager/blob/master/pyramid.py

   and modified for AMi_Image_Analysis

Created on Mon Sep 21 09:25:59 2020

@author: ludovic
"""
import os
import sys
import time
import re
from pathlib import Path
import numpy as np
from scipy import ndimage
import cv2
import multiprocessing
import argparse

MAX_CPU = None  # set to desired integer if needed ie MAX_CPU="8" (keep the "")
# Define below the name of the folder containing unstacked Z images.
_RAWIMAGES = "rawimages"

__version__ = "0.1"
__date__ = "20-05-2021"
__license__ = "New BSD http://www.opensource.org/licenses/bsd-license.php"

def generating_kernel(a):
    kernel = np.array([0.25 - a / 2.0, 0.25, a, 0.25, 0.25 - a / 2.0])
    return np.outer(kernel, kernel)

def reduce_layer(layer, kernel=generating_kernel(0.4)):
    if len(layer.shape) == 2:
        convolution = convolve(layer, kernel)
        return convolution[::2, ::2]

    ch_layer = reduce_layer(layer[:, :, 0])
    next_layer = np.zeros(list(ch_layer.shape) +
                          [layer.shape[2]], dtype=ch_layer.dtype)
    next_layer[:, :, 0] = ch_layer

    for channel in range(1, layer.shape[2]):
        next_layer[:, :, channel] = reduce_layer(layer[:, :, channel])

    return next_layer

def expand_layer(layer, kernel=generating_kernel(0.4)):
    if len(layer.shape) == 2:
        expand = np.zeros(
            (2 * layer.shape[0], 2 * layer.shape[1]), dtype=np.float64)
        expand[::2, ::2] = layer
        convolution = convolve(expand, kernel)
        return 4.0 * convolution

    ch_layer = expand_layer(layer[:, :, 0])
    next_layer = np.zeros(list(ch_layer.shape) +
                          [layer.shape[2]], dtype=ch_layer.dtype)
    next_layer[:, :, 0] = ch_layer

    for channel in range(1, layer.shape[2]):
        next_layer[:, :, channel] = expand_layer(layer[:, :, channel])

    return next_layer

def convolve(image, kernel=generating_kernel(0.4)):
    return ndimage.convolve(image.astype(np.float64), kernel, mode='mirror')

def gaussian_pyramid(images, levels):
    pyramid = [images.astype(np.float64)]
    num_images = images.shape[0]

    while levels > 0:
        next_layer = reduce_layer(pyramid[-1][0])
        next_layer_size = [num_images] + list(next_layer.shape)
        pyramid.append(np.zeros(next_layer_size, dtype=next_layer.dtype))
        pyramid[-1][0] = next_layer
        for layer in range(1, images.shape[0]):
            pyramid[-1][layer] = reduce_layer(pyramid[-2][layer])
        levels = levels - 1

    return pyramid

def laplacian_pyramid(images, levels):
    gaussian = gaussian_pyramid(images, levels)

    pyramid = [gaussian[-1]]
    for level in range(len(gaussian) - 1, 0, -1):
        gauss = gaussian[level - 1]
        pyramid.append(np.zeros(gauss.shape, dtype=gauss.dtype))
        for layer in range(images.shape[0]):
            gauss_layer = gauss[layer]
            expanded = expand_layer(gaussian[level][layer])
            if expanded.shape != gauss_layer.shape:
                expanded = expanded[:gauss_layer.shape[0],
                                    :gauss_layer.shape[1]]
            pyramid[-1][layer] = gauss_layer - expanded

    return pyramid[::-1]

def collapse(pyramid):
    image = pyramid[-1]
    for layer in pyramid[-2::-1]:
        expanded = expand_layer(image)
        if expanded.shape != layer.shape:
            expanded = expanded[:layer.shape[0], :layer.shape[1]]
        image = expanded + layer

    return image

def get_probabilities(gray_image):
    levels, counts = np.unique(gray_image.astype(np.uint8), return_counts=True)
    probabilities = np.zeros((256,), dtype=np.float64)
    probabilities[levels] = counts.astype(np.float64) / counts.sum()
    return probabilities

def entropy(image, kernel_size):
    def _area_entropy(area, probabilities):
        levels = area.flatten()
        return -1.0 * (levels * np.log(probabilities[levels])).sum()

    probabilities = get_probabilities(image)
    pad_amount = int((kernel_size - 1) / 2)
    padded_image = cv2.copyMakeBorder(
        image, pad_amount, pad_amount, pad_amount, pad_amount, cv2.BORDER_REFLECT101)
    entropies = np.zeros(image.shape[:2], dtype=np.float64)
    offset = np.arange(-pad_amount, pad_amount + 1)
    for row in range(entropies.shape[0]):
        for column in range(entropies.shape[1]):
            area = padded_image[row + pad_amount + offset[:,
                                                          np.newaxis], column + pad_amount + offset]
            entropies[row, column] = _area_entropy(area, probabilities)

    return entropies

def deviation(image, kernel_size):
    def _area_deviation(area):
        average = np.average(area).astype(np.float64)
        return np.square(area - average).sum() / area.size

    pad_amount = int((kernel_size - 1) / 2)
    padded_image = cv2.copyMakeBorder(
        image, pad_amount, pad_amount, pad_amount, pad_amount, cv2.BORDER_REFLECT101)
    deviations = np.zeros(image.shape[:2], dtype=np.float64)
    offset = np.arange(-pad_amount, pad_amount + 1)
    for row in range(deviations.shape[0]):
        for column in range(deviations.shape[1]):
            area = padded_image[row + pad_amount + offset[:,
                                                          np.newaxis], column + pad_amount + offset]
            deviations[row, column] = _area_deviation(area)

    return deviations

def get_fused_base(images, kernel_size):
    layers = images.shape[0]
    entropies = np.zeros(images.shape[:3], dtype=np.float64)
    deviations = np.copy(entropies)
    for layer in range(layers):
        gray_image = cv2.cvtColor(images[layer].astype(
            np.float32), cv2.COLOR_BGR2GRAY).astype(np.uint8)
        probabilities = get_probabilities(gray_image)
        entropies[layer] = entropy(gray_image, kernel_size)
        deviations[layer] = deviation(gray_image, kernel_size)

    best_e = np.argmax(entropies, axis=0)
    best_d = np.argmax(deviations, axis=0)
    fused = np.zeros(images.shape[1:], dtype=np.float64)

    for layer in range(layers):
        fused += np.where(best_e[:, :, np.newaxis] == layer, images[layer], 0)
        fused += np.where(best_d[:, :, np.newaxis] == layer, images[layer], 0)

    return (fused / 2).astype(images.dtype)

def fuse_pyramids(pyramids, kernel_size):
    fused = [get_fused_base(pyramids[-1], kernel_size)]
    for layer in range(len(pyramids) - 2, -1, -1):
        fused.append(get_fused_laplacian(pyramids[layer]))

    return fused[::-1]

def get_fused_laplacian(laplacians):
    layers = laplacians.shape[0]
    region_energies = np.zeros(laplacians.shape[:3], dtype=np.float64)

    for layer in range(layers):
        gray_lap = cv2.cvtColor(laplacians[layer].astype(
            np.float32), cv2.COLOR_BGR2GRAY)
        region_energies[layer] = region_energy(gray_lap)

    best_re = np.argmax(region_energies, axis=0)
    fused = np.zeros(laplacians.shape[1:], dtype=laplacians.dtype)

    for layer in range(layers):
        fused += np.where(best_re[:, :, np.newaxis] ==
                          layer, laplacians[layer], 0)

    return fused

def region_energy(laplacian):
    return convolve(np.square(laplacian))

def get_pyramid_fusion(images, min_size=32):

    smallest_side = min(images[0].shape[:2])
    depth = int(np.log2(smallest_side / min_size))
    kernel_size = 5

    pyramids = laplacian_pyramid(images, depth)
    fusion = fuse_pyramids(pyramids, kernel_size)

    return collapse(fusion)

def stack_focus(images, path, name, pyramid_min_size=32, kernel_size=5, blur_size=5, smooth_size=32):

    images = np.array(images, dtype=images[0].dtype)
    aligned_images = align(images)

    stacked_image = get_pyramid_fusion(aligned_images, pyramid_min_size)
    merged = cv2.convertScaleAbs(stacked_image)
    cv2.imwrite(path + name + ".jpg", merged)

def align(images, iterations=1, epsilon=1e-10):
    def _get_homography(image_1, image_2):
        warp_matrix = np.eye(3, 3, dtype=np.float32)
        criteria = (cv2.TERM_CRITERIA_EPS |
                    cv2.TERM_CRITERIA_COUNT, iterations, epsilon)
        # _, homography = cv2.findTransformECC(image_1, image_2, warp_matrix, cv2.MOTION_HOMOGRAPHY, criteria) #OK with openCV 4.0.1
        # Needed Changes for openCV >= 4.2 below
        _, homography = cv2.findTransformECC(
            image_1, image_2, warp_matrix, cv2.MOTION_HOMOGRAPHY, criteria, None, 5)
        return homography

    def _warp(image, shape, homography):
        return cv2.warpPerspective(image, homography, shape, flags=cv2.INTER_LINEAR + cv2.WARP_INVERSE_MAP)

    def _convert_to_grayscale(image):
        return cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)

    gray_images = np.zeros(images.shape[:-1], dtype=np.uint8)
    gray_image_shape = gray_images[0].shape[::-1]

    aligned_images = np.zeros(images.shape, dtype=images.dtype)

    aligned_images[0] = images[0]
    gray_images[0] = _convert_to_grayscale(images[0])
    for index in range(1, images.shape[0]):
        image2_gray = _convert_to_grayscale(images[index])
        homography = _get_homography(gray_images[0], image2_gray)

        gray_images[index] = _warp(image2_gray, gray_image_shape, homography)
        aligned_images[index] = _warp(
            images[index], gray_image_shape, homography)

    return aligned_images

def MERGE_Zstack2(well, file_list, imageDir, outputpath):
    '''file_list only contains images from a single well'''
    images = []
    # count=0
    if len(file_list) != 0:
        for _file in file_list:
            # count+=1
            image = cv2.imread(imageDir + '/' + _file, cv2.IMREAD_COLOR)
            images.append(image)
        stack_focus(images, outputpath + '/', name="%s" % well)
        print("Merged File for well %s saved to %s" %
              (well, outputpath + '/' + well + ".jpg"))
        images.clear()

def natural_sort_key(s):
    global _nsre
    return [int(text) if text.isdigit() else text.lower()
            for text in re.split(_nsre, s)]

def Calculate(directory):
    global Ext, total_wells, MAX_CPU, nproc
    order = []
    for file in os.listdir(directory):
        if os.path.splitext(file)[1] in Ext:
            order.append(file)
    if len(order) == 0:
        print("No Images found: No data? file permission issue?")
        return
    order.sort(key=natural_sort_key)

    if order[0].split('_')[0][-1] in ['a', 'b', 'c']:
        SUBWELL = True
    else:
        SUBWELL = False
    print("TYPE of DATA with SUBWELLS: %s" % SUBWELL)

    if SUBWELL == True:
        _wells = []
        for i in range(len(total_wells)):
            _wells.append(total_wells[i])

    if SUBWELL == False:
        filtered = []
        for i in range(len(total_wells)):
            if total_wells[i][:-1] not in filtered:
                filtered.append(total_wells[i][:-1])
        _wells = filtered

    outputpath = Path(directory).parent

    args = []
    for i in _wells:
        # arg=i, order, str(directory), str(outputpath)
        imagesToStack = [_file for _file in order if i == _file.split('_')[0]]
        arg = i, imagesToStack, str(directory), str(outputpath)
        if len(imagesToStack) != 0:
            args.append(arg)
        # print("ARGUMENT TO PASS: ", arg)

    njobs = len(args)

    if MAX_CPU == 1 or nproc == 1:
        number_processes = 1
    elif njobs >= MAX_CPU and MAX_CPU < nproc:
        number_processes = MAX_CPU
    elif njobs >= MAX_CPU and MAX_CPU == nproc:
        number_processes = nproc-1
    elif njobs < MAX_CPU:
        number_processes = njobs

    print("Number of CORES = ", nproc, "| Number of processes= ",
          number_processes, "| Number of wells to process :", njobs)
    time_start = time.perf_counter()
    pool = multiprocessing.Pool(number_processes)
    results = [pool.apply_async(MERGE_Zstack2, arg) for arg in args]
    pool.close()
    pool.join()
    time_end = time.perf_counter()

    print(f"\nOperation performed in {time_end - time_start:0.2f} seconds")

    last = order[-1].split("_")[0]+".jpg"
    if Path(outputpath).joinpath(last).is_file():
        with open(str(Path(outputpath).joinpath("DONE")), 'w') as f:
            pass

    # Clean up
    del results, order, _wells

########################################################################################################################

def main(args=None):
    global total_wells, nproc
    dirName = os.getcwd()

    parser = argparse.ArgumentParser(prog=__name__,
                                     description='Tool to find and  '
                                     'process all unprocessed '
                                     'plates. ')
    parser.add_argument('dir', metavar='dirName', nargs='?',
                        help='Directory to iterate over.')
    parser.add_argument('--dry', default=False, action='store_true',
                        dest='dry', help='if using --dry '
                        'Only checking, no merging ')
    parser.add_argument('-nproc', metavar='nproc', type=int,
                        help='Set the MAX number of parallel jobs '
                        'to limit RAM usage')

    options = parser.parse_args(args)
    # print("options ", options)

    if options.dir == "." or options.dir == None:
        dirName = os.getcwd()
    else:
        dirName = Path(options.dir)
    if options.nproc is not None:
        global MAX_CPU
        MAX_CPU = options.nproc

    if MAX_CPU != None:
        try:
            int(MAX_CPU)
            if int(MAX_CPU) >= nproc:
                MAX_CPU = nproc  # MAX_CPU cannot be greater than nproc
        except:
            print("ABORTING, nproc not set properly")
            sys.exit()
    else:
        MAX_CPU = nproc-1  # if MAX_CPU is initially None

    # Get the list of all files in directory tree at given path
    listOfDirs = list()
    for (dirpath, dirnames, filenames) in os.walk(dirName):
        # listOfDirs += [os.path.join(dirpath, file) for file in os.listdir(dirpath) if os.path.isdir(os.path.join(dirpath,file))]
        if _RAWIMAGES in dirnames:
            listOfDirs += [os.path.join(dirpath, file) for file in os.listdir(
                dirpath) if os.path.isdir(os.path.join(dirpath, file))]

    rawdata_subfolders = list()
    for elem in listOfDirs:
        if os.path.islink(elem) is False:
            directory = Path(elem)
            parents = directory.parents
            if "Image_Data" in str(directory):
                continue
            if directory.parts[-1] == _RAWIMAGES and not Path(parents[0]).joinpath("DONE").is_file():
                rawdata_subfolders += [Path(parents[0]).joinpath(_RAWIMAGES)]
            elif directory.parts[-1] == _RAWIMAGES and Path(parents[0]).joinpath("DONE").is_file():
                print(
                    f"Data in {directory} seem to have already been processed, skipping")
    del listOfDirs

    # To Fix multiprocessing issue with OSX Catalina
    if sys.platform == 'darwin' and multiprocessing.get_start_method() != 'forkserver':
        multiprocessing.set_start_method('forkserver', force=True)

    if len(rawdata_subfolders) == 0:
        print("\nNo data to merge, it seems everything has already been processed.\n")
    for _dir in rawdata_subfolders:
        print("Merging Z-stacks for directory: ", _dir)
        if options.dry is False:
            Calculate(_dir)
            time.sleep(2)

    # Clean up
    for i in total_wells:
        del i
    del total_wells

if __name__ == '__main__':
    ##################################### GLOBAL VARIABLES #################################################################
    Ext = [".tif", ".tiff", ".TIFF", ".jpg",
           ".jpeg", ".JPG", ".JPEG", ".png", ".PNG"]
    rows = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H']
    cols = ['1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12']
    wells = ['a', 'b', 'c']
    _nsre = re.compile('([0-9]+)')

    total_wells = [row + str(col) + str(well)
                   for row in rows for col in cols for well in wells]

    nproc = multiprocessing.cpu_count()
########################################################################################################################
    main()