#! /usr/bin/env python
try:
	import sys
	import logging
	logging.getLogger("scapy.runtime").setLevel(logging.ERROR)
	from scapy.all import *
	import re
	import os
	import string
	import argparse
	import fileinput
	import types
	import datetime
	from time import gmtime, strftime
except:
	sys.stderr.write("ERROR: nepodarilo se importovat vsechny potrebne knihovny\n")
	exit(1)

# napoveda
def napoveda ():
	napoveda = """\n\t======================== [ NAPOVEDA PROGRAMU ] ==================================
	|\t
	|\tVitejte v napovede programu SIPSCAN do predmetu ISA
	|\tAutor: Tomas Slunsky, xsluns01@stud.fit.vutbr.cz
	|\t
	|\t[ UZITI ]
	|\t	./sipscan.py {-f|-i} name -o file [-p number]
	|\t
	|\t[ PARAMETRY ]
	|\t-f [--file] name -- data pro analyzu se ziskaji ze souboru formatu pcap
	|\t-i [--interface] name -- data sa odchyti z rozhrani zo zadanym nazvom
	|\t-o [--output] file -- vysledky sa zapisu do souboru so zadanym nazvom
	|\t-p [--port] num -- cislo portu na kterem probiha signalizacea SIP
	|
	=================================================================================\n"""
	return napoveda

# err report
def error(msg,errcode):
	sys.stderr.write("ERROR: "+msg+"\n")
	exit(errcode)

# zisk URI
def getUri(line):
	if re.search(r"(?<=\<)[^>]+(?=\>)",line):
		v = (re.search(r"(?<=\<)[^>]+(?=\>)",line)).group(0)
		v = v.replace("sip:","")
		v = v.replace("sips:","")
		return v
	else:
		return line.replace("sip:","")

# vyparsovani dulezitych dat z paketu invite
def pktParser(pkt):
	# init
	invitePacked = {}

	# zpracovani SIP
	if pkt.haslayer(Raw):
		# nacteni obsahu paketu a ODSTRRANENI \'
		paket = (repr(pkt[Raw].load)).replace("'","")

		# zpracovani paketu
		uri   	   = re.search(r"(?<=uri\=[\"\'])\s*[^\"]+(?=\")", paket)
		bye   	   = re.search(r"(?<=BYE\ssip:)\s*[^\s]+(?=\s)", paket)
		ack   	   = re.search(r"(?<=ACK\ssip:)\s*[^\s]+(?=\s)", paket)
		register   = re.search(r"(?<=REGISTER\ssip:)\s*[^\s]+(?=\s)", paket)
		invite     = re.search(r"(?<=INVITE\ssip:)[\w]+@[\w]+.[a-zA-Z]+", paket)
		udp        = re.search(r"(?<=UDP\s)[\w\.]+(?=;)", paket)
		to         = re.search(r"(?<=[tT][oO]:\s)([\w\s\"\']+)?<[^>]+>", paket)
		fromP      = re.search(r"(?<=From:\s)([\"\'\w\s]+)?<[^>]+>", paket)
		contact    = re.search(r"(?<=Contact:\s)([\"\'\w\s]+)?<[^>]+>", paket)
		realm      = re.search(r"(?<=realm=[\"\'])\s*[^\"\']+(?=[\"\'])", paket)
		username   = re.search(r"(?<=username=[\"\'])\s*[^\"\']+(?=[\"\'])", paket)

		# nahazeni do slovniku
		if register is not None: invitePacked["register"] 	= register.group(0)
		if invite 	is not None: invitePacked["invite"] 	= invite.group(0)
		if ack 	 	is not None: invitePacked["ack"] 		= ack.group(0)
		if bye 		is not None: invitePacked["bye"] 		= bye.group(0)
		if udp 		is not None: invitePacked["udp"] 		= udp.group(0)
		if to 		is not None: invitePacked["to"] 		= getUri(to.group(0))
		if fromP 	is not None: invitePacked["from"] 		= getUri(fromP.group(0))
		if contact  is not None: invitePacked["contact"]	= getUri(contact.group(0))
		if uri 		is not None: invitePacked["uri"] 		= getUri(uri.group(0))
		if realm 	is not None: invitePacked["realm"]		= realm.group(0)
		if username is not None: invitePacked["username"]	= username.group(0)

	# zpracovani dat  z IP vrstvy
	if pkt.haslayer(IP):
		# zakladni parsovani se scapy
		invitePacked["source"]		= pkt[IP].src
		invitePacked["destination"] = pkt[IP].dst

	# zpracovani dat  z UDP vrstvy
	if pkt.haslayer(UDP):
		# zakladni parsovani se scapy
		invitePacked["sourcePort"]		= pkt[UDP].sport
		invitePacked["destinationPort"] = pkt[UDP].dport

	# zpracovani dat  z TCP vrstvy
	elif pkt.haslayer(TCP):
		# zakladni parsovani se scapy
		invitePacked["sourcePort"]		= pkt[TCP].sport
		invitePacked["destinationPort"] = pkt[TCP].dport

	# zpracovani dat  z SCTP vrstvy
	elif pkt.haslayer(SCTP):
		# zakladni parsovani se scapy
		invitePacked["sourcePort"]		= pkt[SCTP].sport
		invitePacked["destinationPort"] = pkt[SCTP].dport

	#navrat
	return invitePacked

# srovnam dva seznamy a vratim shodne elemnty
def comp2list(lis1,lis2):
	new = []
	for el in lis1:
		if el in lis2:
			new.append(el)
	return new

def parseAMCOfSDP(param,mode="a",m="audio"):
	ret = {}

	# zisk kodeku z SDP
	if mode == "a":
		i = 1
		for element in param:
			payload_type = re.search(r'(?<=:)[0-9]+(?=\s)',element)
			name = re.search(r'(?<=[0-9]\s).*',element)

			if "payload-type" not in ret.keys():
				if payload_type is not None:
					ret["payload-type"] = payload_type.group(0)

				if name is not None:
					ret["name"] = name.group(0)
			else:
				if payload_type is not None:
					ret["payload-type"+str(i)] = payload_type.group(0)

				if name is not None:
					ret["name"+str(i)] = name.group(0)
				i=i+1
		return ret

	# ziskani portu src a dst z SDP
	elif mode=="m":
		if type(param)==str:
			get_port = re.search(r'(?<=\s)[0-9]+(?=\s)',param)

			if get_port is not None:
				return get_port.group(0)
			else:
				return False

		elif type(param)==list:
			if m=="audio":
				get_port = re.search(r'(?<=audio\s)[0-9]+(?=\s)',param[0])

			elif m=="video":
				if len(param)>1:
					get_port = re.search(r'(?<=video\s)[0-9]+(?=\s)',param[1])

			if get_port is not None:
				return get_port.group(0)
			else:
				return False

	# zisk src a dst IP z SDP
	elif mode=="c":
		get_ip = re.search(r'[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+',param)

		if get_ip is not None:
			return get_ip.group(0)
		else:
			return False

	# zisk cisla kodeku a nahazeni do pole
	elif mode=="m2":
		param = param[::-1] # obraceni pro jednoduchost
		get_codecs = re.search(r'([0-9]+\s)+',param)

		if get_codecs is not None:
			ret=[]
			# vydelame prazdne stringy
			ar= (get_codecs.group(0)).split(" ")
			for it in ar:
				if it!="":
					ret.append(int(it[::-1])) # potreba jeste jednou reverse pro puvodni hodnoty
			return ret
		return False

def pktSdpParser(pkt, mode=1):
	# init
	sdp = {}
	ret = []
	ret2 = ret3 = ""

	# zpracovani SIP
	if pkt.haslayer(Raw):
		# OREZANI \' A nacteni obsahu paketu
		load = (repr(pkt[Raw].load)[1:-1]).replace("\\r\\n","#")

		# parsovani
		media		=	re.findall(r'(?<=a=)\s*[^\#]+(?=\#)',load)
		relation	=	re.findall(r'(?<=\#m=)\s*[^\#]+(?=\#)',load)
		adress		=	re.search(r'(?<=\#o=)\s*[^\#]+(?=\#)',load)

		# zpracovani
		if media is not None:
			ret = media

		if relation is not None:
			ret2 = relation#relation.group(0)

		if adress is not None:
			ret3 = adress.group(0)

		# co vratit
		if mode==1: return ret
		elif mode==2: return ret2
		elif mode==3: return ret3

# vyhodnoceni Paketu
def executePkts(pkts):
	retlist = []
	tmplist = {}
	registers = {}
	newret  = []
	data = {}
	zacatek_hovoru = 0  # prvni invite
	odpoved_na_hovor = 0 # prvni invite

	# zpracovani hovoru
	for index in range (len(pkts)):
		if pkts[index] and (Raw in pkts[index]):
			if pktSearch(pkts[index],"REGISTER"):
				#print "skacu do REGISTER"
				offset=1
				while (1):
					# pokud odpoved bude 1(trying) nebo 3(continue) nacitam dalsi odpovedi
					if getAnswer(pkts[index+offset]) in [1,3]:
						offset = offset + 1
						continue

					# pokud odpoved je 4,5 nebo 6, znamena to preruseni nebo chybu a je jasne ze
					# registrace probehne znova, takze break
					if getAnswer(pkts[index+offset]) in [4,5,6]:
						break

					# pokud registrace probehla uspesne, zpracuju data o registraci
					if getAnswer(pkts[index+offset]) == 2:
						tmplist = pktParser(pkts[index])
						tmplist["timestamp"] = pkts[index+offset].time
						data=addDictToDict("REGISTER",tmplist,data)
						break

					if offset > len(pkts)+50:break

				# jump to index+offset => index je paket ktrey proveruji + preskocim
				# ty odpovedi, ktery prisly na invite coz je ten offset
				index=index+offset
				continue

			if pktSearch(pkts[index],"INVITE"):
				offset=1 # posun v poli paketu

				# prisel prvni invite, zaznamenam zacatek hovoru do promenne
				if (zacatek_hovoru==0):
					zacatek_hovoru = pkts[index].time

				# zpraxovani hovoru
				while(1):
					# V PRIPADE ZE PRISEL cancel = UKONCENI
					if pktSearch(pkts[index+offset],"CANCEL"):
						# ZPRAOVANI tj naparsovani dat o hovoru
						tmplist = pktParser(pkts[index])
						tmplist["timestamp_start"] = zacatek_hovoru
						tmplist["timestamp_answer"] = pkts[index+offset].time
						tmplist["timestamp_end"] = pkts[index+offset].time
						tmplist["rtp_src_port"] = parseAMCOfSDP(pktSdpParser(pkts[index],2),"m")
						tmplist["rtp_src_ip"] = parseAMCOfSDP(pktSdpParser(pkts[index],3),"c")
						#tmplist["rtp_dst_port"] = ""
						#tmplist["rtp_dst_ip"] = ""
						data=addDictToDict("INVITE",tmplist,data)
						break

					if offset > len(pkts)+50:break

					# pokud odpoved bude 1(trying) nebo 3(continue) nacitam dalsi odpovedi
					if getAnswer(pkts[index+offset]) in [1,3]:
						#print "Tryin nebo continue, pokracuju\r\n"
						offset = offset + 1
						continue

					if getAnswer(pkts[index+offset]) == False:
						offset = offset + 1
						continue

					# pokud odpoved je 4,5 nebo 6, znamena to preruseni nebo chybu a je jasne ze
					# registrace probehne znova, takze break
					if getAnswer(pkts[index+offset]) in [4,5,6]:
						# overim zdali, ma INVITE nejaky dalsi zadosti, pokud ne budu to povazovat za ukonecnej hovor
						if pktReqSearch(pkts,(index+offset),"INVITE"):
							break # pokud tam jeste INVITE JE, pouze breaknu
						# pokud ale dale neni jiz invite, hovor zrejme skoncil
						else:
							# konec hovoru - kazdopadne bytam mela byt jeste ACK
							konec_hovoru = 0 # init
							tmplist = pktParser(pkts[index])
							tmplist["timestamp_start"] = zacatek_hovoru
							tmplist["timestamp_answer"] = pkts[index+offset].time
							tmplist["rtp_src_port"] = parseAMCOfSDP(pktSdpParser(pkts[index],2),"m")
							tmplist["rtp_src_ip"] = parseAMCOfSDP(pktSdpParser(pkts[index],3),"c")
							tmplist["timestamp_end"] = pkts[index+offset].time
							data=addDictToDict("INVITE",tmplist,data)
						break

					# pokud registrace probehla uspesne, zpracuju data o registraci
					if getAnswer(pkts[index+offset]) == 2:
						#print "invite byl uspesny, parsuju data\r\n"
						konec_hovoru = 0 # init
						tmplist = pktParser(pkts[index])
						tmplist["timestamp_start"] = zacatek_hovoru
						tmplist["timestamp_answer"] = pkts[index+offset].time

						# zpracovani SDP protokolu
						####### ==>>>>> zpracovani kodeku <<<<<<==== #########
						client = pktSdpParser(pkts[index])		# sem se nacte cely pole Acek

						# PRO AUDIO
						try:
							kodeky_klienta_audio = parseAMCOfSDP(pktSdpParser(pkts[index],2)[0],"m2")
							kodeky_serveru_audio = parseAMCOfSDP(pktSdpParser(pkts[index+offset],2)[0],"m2")
							matched_codecs_audio = comp2list(kodeky_klienta_audio,kodeky_serveru_audio)

							# PRO VIDEO
							# existuje audio i video u klienta a serveru?
							if len(pktSdpParser(pkts[index],2))==2 and len(pktSdpParser(pkts[index+offset],2))==2:
								kodeky_klienta_video = parseAMCOfSDP(pktSdpParser(pkts[index],2)[1],"m2")
								kodeky_serveru_video = parseAMCOfSDP(pktSdpParser(pkts[index+offset],2)[1],"m2")
								matched_codecs_video = comp2list(kodeky_klienta_video,kodeky_serveru_video)
						except:pass

						match_audio=[]
						match_video=[]

						# zde beru jeden obsah atrbitu a ze SDP a zjistuji zdali obsahuje payload z odpovedi SIP 200
						try:
							for a in client:
								# pokud ano, vim ze se ma ten kodek pouzit a pridam ho
								if re.search(r"(?<=:)\w+(?=\s)",a):
									if re.search(r"fmtp",a): continue #x skip, neni kodek
									anum = int((re.search(r"(?<=:)\w+(?=\s)",a)).group(0))
									for mc in matched_codecs_audio:
										if mc == anum:
											match_audio.append(a)

									# existuje audio i video u klienta a serveru?
									if len(pktSdpParser(pkts[index],2))==2 and len(pktSdpParser(pkts[index+offset],2))==2:
										for mc2 in matched_codecs_video:
											if mc2 == anum:
												match_video.append(a)
						except:pass

						if match_audio:
							# z tech shodnych vyparsuju informace o koduku
							match_audio = parseAMCOfSDP(match_audio)

							# prepisu shodna pole do tmp listu
							for akey in match_audio.keys():
								tmplist[akey] = match_audio[akey]

						if match_video:
							# z tech shodnych vyparsuju informace o koduku
							match_video = parseAMCOfSDP(match_video)

							# prepisu shodna pole do tmp listu
							for akey in match_video.keys():
								tmplist[akey+"_video"] = match_video[akey]

						# prochazim dal paketama az po BYE abych ziskal cas konce hovoru
						posun = offset
						while (1):
							posun = posun + 1
							if (index+posun)<=(len(pkts)-1):
								if pktSearch(pkts[index+posun],"BYE"):
									konec_hovoru = pkts[index+posun].time
									break

							if (posun>len(pkts)):
								break # fatal error

						# zapisu rtp data ze SDP prtookolu do tmplistu
						try:
							tmplist["rtp_src_port"] = parseAMCOfSDP(pktSdpParser(pkts[index],2),"m")
							tmplist["rtp_dst_port"] = parseAMCOfSDP(pktSdpParser(pkts[index+offset],2),"m")
						except:pass

						try:
							if len(pktSdpParser(pkts[index],2))==2 and len(pktSdpParser(pkts[index+offset],2))==2:
								tmplist["rtp_src_port_video"] = parseAMCOfSDP(pktSdpParser(pkts[index],2),"m","video")
								tmplist["rtp_dst_port_video"] = parseAMCOfSDP(pktSdpParser(pkts[index+offset],2),"m","video")
						except:pass

						try:
							tmplist["rtp_src_ip"] = parseAMCOfSDP(pktSdpParser(pkts[index],3),"c")
							tmplist["rtp_dst_ip"] = parseAMCOfSDP(pktSdpParser(pkts[index+offset],3),"c")
						except:pass

						tmplist["timestamp_end"] = konec_hovoru
						data=addDictToDict("INVITE",tmplist,data)
						break

					if getAnswer(pkts[index+offset]) == False:
						offset = offset + 1
						continue

				# jump to index+offset => index je paket ktrey proveruji + preskocim
				# ty odpovedi, ktery prisly na invite coz je ten offset
				index=index+offset
				continue
	return data

# prevod paketu do XML
def pktsToXML(data):
	output = "<sipscan>\r\n"

	# zpracovani klicu
	for key in data.keys():
		if re.match(r"REGISTER\w*",key):
			output = output +"\t<registration>\r\n"

			if "destination" in data[key].keys() and "uri" in data[key].keys():
				output = output +"\t\t<registratar ip=\""+data[key]["destination"]+"\" uri=\""+data[key]["uri"]+"\" />\r\n"

			if "source" in data[key].keys() and "from" in data[key].keys():
				output = output +"\t\t<user-agent ip=\""+data[key]["source"]+"\" uri=\""+data[key]["from"]+"\">\r\n"

			if "username" in data[key].keys() and "realm" in data[key].keys() and "uri" in data[key].keys():
				output = output +"\t\t<authentication username=\""+data[key]["username"]+"\" realm=\""+data[key]["realm"]+"\" uri=\""+data[key]["uri"]+"\" />\r\n"

			if "timestamp" in data[key].keys():
				output = output +"\t\t<time registration=\""+getTimeFromTStamp(data[key]["timestamp"])+"\" />\r\n"

			output = output +"\t</registration>\r\n"

		if re.match(r"INVITE\w*",key):
			output = output +"\t<call>\r\n"
			if "source" in data[key].keys() and "from" in data[key].keys():
				output = output +"\t\t<caller ip=\""+data[key]["source"]+"\" uri=\""+data[key]["from"]+"\" />\r\n"

			if "destination" in data[key].keys() and "to" in data[key].keys():
				output = output +"\t\t<callee ip=\""+data[key]["destination"]+"\" uri=\""+data[key]["to"]+"\" />\r\n"

			if "timestamp_start" in data[key].keys() and "timestamp_answer" in data[key].keys() and "timestamp_end" in data[key].keys():
				output = output +"\t\t<time start=\""+getTimeFromTStamp(data[key]["timestamp_start"])+"\" answer=\""+getTimeFromTStamp(data[key]["timestamp_answer"])+"\" end=\""+getTimeFromTStamp(data[key]["timestamp_end"])+"\" />\r\n"

			output = output +"\t\t<rtp>\r\n"

			if "rtp_src_ip" in data[key].keys() and "rtp_src_port" in data[key].keys():
				output = output +"\t\t\t<caller ip=\""+data[key]["rtp_src_ip"]+"\" port=\""+data[key]["rtp_src_port"]+"\" />\r\n"

			if "rtp_dst_ip" in data[key].keys() and "rtp_dst_port" in data[key].keys():
				output = output +"\t\t\t<callee ip=\""+data[key]["rtp_dst_ip"]+"\" port=\""+data[key]["rtp_dst_port"]+"\" />\r\n"

			if "payload-type" in data[key].keys() and "name" in data[key].keys():
				output = output +"\t\t\t<codec payload-type=\""+data[key]["payload-type"]+"\" name=\""+data[key]["name"]+"\" />\r\n"

			if len(re.findall(r"payload-type[0-9]+\s",countOfCols(data[key].keys())))>0:
				for index in range (len(re.findall(r"payload-type\w+",countOfCols(data[key].keys())))):
					output = output +"\t\t\t<codec payload-type=\""+data[key]["payload-type"+str(index+1)]+"\" name=\""+data[key]["name"+str(index+1)]+"\" />\r\n"

			output = output +"\t\t</rtp>\r\n"

			# overeni media description kodeku
			if "rtp_src_port_video" in data[key].keys() and "payload-type_video" in data[key].keys() :
				output = output +"\t\t<rtp>\r\n"

				if "rtp_src_ip" in data[key].keys() and "rtp_src_port_video" in data[key].keys():
					output = output +"\t\t\t<caller ip=\""+data[key]["rtp_src_ip"]+"\" port=\""+data[key]["rtp_src_port_video"]+"\" />\r\n"

				if "rtp_dst_ip" in data[key].keys() and "rtp_dst_port_video" in data[key].keys():
					output = output +"\t\t\t<callee ip=\""+data[key]["rtp_dst_ip"]+"\" port=\""+data[key]["rtp_dst_port_video"]+"\" />\r\n"

				if "payload-type_video" in data[key].keys() and "name_video" in data[key].keys():
					output = output +"\t\t\t<codec payload-type=\""+data[key]["payload-type_video"]+"\" name=\""+data[key]["name_video"]+"\" />\r\n"

				if len(re.findall(r"payload-type[0-9]+_video\s",countOfCols(data[key].keys())))>0:
					for index in range (len(re.findall(r"payload-type[0-9]+_video\s",countOfCols(data[key].keys())))):
						output = output +"\t\t\t<codec payload-type=\""+data[key]["payload-type"+str(index+1)+"_video"]+"\" name=\""+data[key]["name"+str(index+1)+"_video"]+"\" />\r\n"

				output = output +"\t\t</rtp>\r\n"
			output = output +"\t</call>\r\n"
	output = output +"</sipscan>\r\n"
	return output

# overeni protokoli a portu
def checkProtocolAndPort (pkt,protocols,port,mode=1):
	# jake porty filtrovat
	if type(port) is list:
		ports = port
	elif type(port) is int:
		ports = [port]
	else:
		error("spatny port",1)

	# prochazim filtrovane protokoly
	for protocol in protocols:
		if mode==1:
			# zjistuju zdali je dany paket daneho protokolu
			if pkt.haslayer(protocol):
				# zjistuju zdali je dany paket daneho protokolu # jeste treba zjistit, jestli tam ma byt and
				if pkt[protocol].sport in ports or pkt[protocol].dport in ports:
					return True
				else:
					return False
		else:
			# zjistuju jen zdali je dany paket daneho protokolu
			if pkt.haslayer(protocol):
				return True
	return False

# vyparsovani vsech paketu ze SIP zprava z paketuu pcap souboru
def filter2 (file, port=5060, bymsg=1):
	# vystupni soubor
	try:
		if os.path.isfile(file) and os.access(file, os.R_OK):
			pkts = rdpcap(file)
		else:
			error("Soubor neexistuje, nebo neni citelny",2)
	except:
		error("chyba",2)

	# jake zpravy a protokoly filtrovat
	messages = ["SIP","INVITE","ACK","BYE","REGISTER","CANCEL"]
	protocols = [TCP,UDP,SCTP] # nad jakejma protokolama vetsinou bezi SIP

	# navratove pole
	retlist = []

	# prochazim paket po paketu
	#for pkt in pkts:
	for index in range (len(pkts)):
		# filtr, ktery vyhodi jine pakate nez na povolenych protokolech a portech
		if not checkProtocolAndPort(pkts[index],protocols,port):
			continue

		# samotna zprava paketu se nahraje do pole
		if (pkts[index]).haslayer(Raw):

			# nacteni obsahu paketu + orezani apostrofu
		 	load = (repr((pkts[index])[Raw].load)).replace("'","")

		 	# projit zpravy po zprave
		 	for message in messages:
		 		#zjisteni zdali se jedna o paket sip
		 		if re.match(r'^'+message,load):
		 			# v pripade ze shody pridam do vysledneho pole
		 			retlist.append((pkts[index])) # retlist.append(load)
		 			break # nactu dalsi paket
	return retlist

# prevod casu
def getTimeFromTStamp (timestamp):
	t=datetime.datetime.fromtimestamp(round(int(timestamp)))
	time=t.strftime("%Y-%m-%dT%H:%M:%S")
	return time

# vyparsovani navratoveho kodu z paketu
def getAnswer(pkt,mode=1):
	if pkt.haslayer(Raw):
		# nacteni obsahu paketu + odstraneni apostrofu
		load = (repr(pkt[Raw].load)).replace("'","")

		# vyprasuju z paketu SIP s verzi a navratovym kodem
		match = re.search(r"SIP\/[0-9]+\.[0-9]+\s[0-9]{3,3}", load)
		if match is None:
			return False
		else:
			# pokud ho najdu, vyparsuju z nej samotnou prvni cislici a prevedu na int
			match_int = re.search(r"(?<=\s)[0-9]{3,3}", match.group(0))
			if match_int is not None:
				num_of_answer = match_int.group(0)
				if mode==1:
					return int(num_of_answer[0:1]) ## vracim prvni cislo chyby
				else:
					return int(num_of_answer[0:3]) ## vracim celou chybu
			else:
				return False
	else:
		return False

# overi zdali je v paketu zprava na zacatku
def pktSearch(pkt,msg):
	if pkt.haslayer(Raw):
		# nacteni obsahu paketu
		load = (repr(pkt[Raw].load)).replace("'","")
		if re.match(r'^'+msg,load):
			return True
		else:
			return False
	else:
		return False

# overi zdali se v nasledujicih paketech vyskytuje nejakY SIP REQUEST
def pktReqSearch(pkts,index,req):
	for i in range ((len(pkts))-(index)):
		if pktSearch(pkts[index+i],req):
			return True
	return False

# pridani slovniku1 do slovniku2  pod klicem key
def addDictToDict(key,dic1,dic2):
	# jestlize seznam pod danym klicem neni obsazen v seznamu
	if key not in dic2.keys():
		# tak jej muzu pridat
		dic2[key]=dic1
	else:
		newKey=1
		while(1):
			if str(key+str(newKey)) not in dic2.keys():
				dic2[str(key+str(newKey))]=dic1
				break
			else:
				newKey=newKey+1
	return dic2

# sniffovaci funkce pro odposlech rozhrani
def sniffIfaceAndPort(interface,port):
	if interface and port:
		# odposlech rozhrani
		ret = ""
		try:
			ret = sniff(iface=interface,filter="(tcp or udp) and port "+str(port)) # +str(port)prn = funkce podle ktere se bude filtrovat
		except:
			error("nepovedlo se odposlechnout pakety",20)
	else:
		return False

	# navrat jako odposlechnute pakety a pote zpracovani jako souboru pcap
	return ret

# keys into string
def countOfCols (columns):
	str_out = ""
	for key in columns:
		str_out = str_out+" "+key
	return str_out

# osetreni kombinaci paramtru apod.
def argsExecute(args):
	# pocet parametru
	for parametr in ["-f","-i","-o","-p","-h","--file","--interface","--output","--port","--help","-fic"]:
		if len(re.findall(r""+parametr+"(?=\s)", countOfCols (sys.argv)))>1:
			error("Nelze zadat jeden parametr vicekrat, dukaz: "+countOfCols (sys.argv),10)

	# osetreni napovedy
	if args.help:
		if len(sys.argv)!=2:
			error("Parametr help nesmi byt kombinovan",11)

	# osetreni zdali nedoslo k zadani soucasne
	if args.file and args.interface:
		error("NELZE ZADAT VSTUP JAK Z ROZHRANI TAK ZE SOUBORU, zvolte jen jeden",12)

	# test existence souboru
	if args.file:
		if not os.path.isfile(args.file) or not os.access(args.file, os.R_OK):
			error("zadany soubor nebyl nalezen, nebo neni citelny",13)

	# osetreni povinnosti parametru
	if not args.file and not args.interface and not args.help:
		error("musi byt zadan alespon jeden z dvojice parametru -i IFACE|-f FILE",14)

	# osetreni povinnosti parametru
	if not args.output and not args.help:
		error("parametr pro vystup musi byt zadan!",15)

	# osetreni povinnosti parametru
	if not args.output:
		error("zadany soubor pro VYSTUP nebyl nalezen, nebo neni zapisovatelny",16)

# zpracovani parametru
arguments = argparse.ArgumentParser(description="Skript do Predmetu ISA, SIPSCAN")
arguments = argparse.ArgumentParser(add_help=False)
arguments.add_argument('--file','-f',action="store", dest="file")
arguments.add_argument('--interface','-i',action="store", dest="interface")
arguments.add_argument('--output','-o', action="store", dest="output")
arguments.add_argument('--port','-p',    action="store", dest="port")
arguments.add_argument("--help", "-h", action="store_true", dest="help")

try:
	args = arguments.parse_args()## naparsovani argumentu
except:
	error("nepovoleny argument",1)

# predvolanim parametru se nejprve osetri
argsExecute(args)

# help
if args.help:
	print(napoveda())

# overeni zdali je zadan port - volitelny, muze a nemusi
if args.port:
	port = int(args.port)
else:
	port = 5060

if args.output:
	try:
		outFile = open(args.output,"wt")
	except:
		error("nepovedlo se otevrit vystupni soubor",16)

if args.interface:
	data = sniffIfaceAndPort(args.interface,port)
	data = executePkts(data)
	xmlData = pktsToXML(data)
	outFile.write(xmlData)
elif args.file:
	f=filter2 (args.file,port)
	data = executePkts(f)
	xmlData = pktsToXML(data)
	outFile.write(xmlData)

exit(0)


# -*- coding: utf-8 -*-

import logging

from cliff.command import Command

class BaseCommand(Command):
    log = logging.getLogger(__name__)
    common_params_set = ['secret_access_key', 'signature_method', 'version',
                         'signature_version', 'access_key_id']

    def get_parser(self, prog_name):
        """Deal with some common arguments """

        parser = super(BaseCommand, self).get_parser(prog_name)
        # API version
        parser.add_argument(
            "--api_version",
            dest="version",
            metavar="<api-version>",
            choices=["1"],
            default="1",
            help="API to use, Valid 1, Default 1",
        )
        parser.add_argument(
            "--secret_access_key",
            required=True,
            metavar="<secret-access-key>",
            help="Secret access key",
        )
        parser.add_argument(
            "--access_key_id",
            required=True,
            metavar="<access-key-id>",
            help="Access Key ID",
        )
        parser.add_argument(
            "--signature_method",
            metavar="<signature-method>",
            choices=["HmacSHA256", "HmacSHA1"],
            default="HmacSHA256",
            help="Signature method, Valid: HmacSHA256, HmacSHA1"
        )
        parser.add_argument(
            "--signature_version",
            metavar="<signature-version>",
            choices=["1"],
            default="1",
            help="Signature version, Valid 1, Default 1",
        )
        return parser

    def take_action(self, parsed_args):
        return


import collections

from django.conf import settings
from django.db import models
from django.utils import timezone
from django.utils.translation import ugettext_lazy as _

from account.models import SignupCode, SignupCodeResult

Member = collections.namedtuple("Member", ["email", "signup_code", "user", "invited", "expired"])

class Cohort(models.Model):

    name = models.CharField(_("name"), max_length=35)
    created = models.DateTimeField(_("created"), default=timezone.now, editable=False)

    class Meta:
        permissions = (
            ("manage_cohorts", "Can manage cohorts"),
        )

    def members(self):
        members = []
        for scc in self.signupcodecohort_set.select_related():
            try:
                scr = SignupCodeResult.objects.get(signup_code=scc.signup_code_id)
            except SignupCodeResult.DoesNotExist:
                user = None
            else:
                user = scr.user
            members.append(
                Member(
                    scc.signup_code.email,
                    scc.signup_code,
                    user,
                    bool(scc.signup_code.sent),
                    timezone.now() > scc.signup_code.expiry
                )
            )
        return members

    def member_counts(self):
        members = self.members()
        return {
            "total": len(members),
            "users": len([m for m in members if m.user is not None]),
            "pending": len([m.signup_code for m in members if not m.invited]),
        }

    def send_invitations(self):
        for sc in [m.signup_code for m in self.members() if not m.invited]:
            sc.send()

    def __str__(self):
        return self.name

class SignupCodeCohort(models.Model):
    """
    fetch cohort of a given signup code
        SignupCodeCohort.objects.select_related(
            "cohort"
        ).get(
            signup_code__code="abc"
        ).cohort

    list of people waiting NOT on the site already or invited
        WaitingListEntry.objects.exclude(
            email__in=SignupCode.objects.values("email")
        ).exclude(
            email__in=User.objects.values("email")
        )
    """
    signup_code = models.OneToOneField(SignupCode, on_delete=models.CASCADE)
    cohort = models.ForeignKey(Cohort, on_delete=models.CASCADE)

class UserCohort(models.Model):
    """
    Upon signup we create an instance of this model associating the new user
    and their cohort
    """
    user = models.OneToOneField(settings.AUTH_USER_MODEL, on_delete=models.CASCADE)
    cohort = models.ForeignKey(Cohort, on_delete=models.CASCADE)


import argparse
import soundfile as sf

import torch

# import class()
from model.model_head import VoiceFilter
from utils.get_audio import GetAudio

# import def()
from dataloader.dataloader import Loader

# init model
model = VoiceFilter()
model.cuda()
model.load_state_dict(torch.load("./weights/checkpoint0.pth"))

train_loader = Loader(batch_size=1, num_workers=0)

data_path = "./audio/"
data_format = ".wav"
get_audio = GetAudio(data_path=data_path, data_format=data_format)

def test():
    model.eval()
    for iter, data in enumerate(train_loader):
        print(iter)
        if iter > 3: break
        refer_spec, clear_spec, noicy_spec = data
        refer_spec, clear_spec, noicy_spec = refer_spec.cuda(), clear_spec.cuda(), noicy_spec.cuda()
        pred_spec = model(refer_spec, noicy_spec)
        noicy_audio = get_audio.inverse_mel(noicy_spec.detach().cpu().numpy()[0, 0])
        clear_audio = get_audio.inverse_mel(clear_spec.detach().cpu().numpy()[0, 0])
        pred_audio = get_audio.inverse_mel(pred_spec.detach().cpu().numpy()[0, 0])
        sf.write(data_path + "noicy_%i" % iter + data_format, noicy_audio, 16000)
        sf.write(data_path + "clear_%i" % iter + data_format, clear_audio, 16000)
        sf.write(data_path + "pred_%i" % iter + data_format, pred_audio, 16000)

if __name__ == "__main__":
    test()


import argparse
import logging
import os
import re
import subprocess
import sys
parser = argparse.ArgumentParser(description="Aggregate all protobuf files")
parser.add_argument(
    "-p",
    "--package_name",
    type=str,
    default="cosmospy_protobuf",
    help=
    "Name for the package to build. This will aggregate all files in the src/{package_name} folder",
)
args = parser.parse_args()

package_name = "src/" + args.package_name
logging.basicConfig(format="%(asctime)s - %(levelname)s:%(message)s",
                    level=logging.DEBUG)
absolute_path = os.path.abspath(package_name)

def run_protoc(filepath):
    if (os.path.basename(filepath) == "query.proto"
            or os.path.basename(filepath) == "service.proto"):
        cmd = [
            sys.executable,
            "-m",
            "grpc_tools.protoc",
            "--proto_path",
            absolute_path,
            "--python_out",
            package_name,
            "--pyi_out",
            package_name,
            "--grpc_python_out",
            package_name,
            "--grpclib_python_out",
            package_name,
            filepath,
        ]
        logging.info(f"Compiling proto and grpc file: {filepath}")
    else:
        cmd = [
            sys.executable,
            "-m",
            "grpc_tools.protoc",
            f"--proto_path={absolute_path}",
            f"--python_out={package_name}",
            f"--pyi_out={package_name}",
            filepath,
        ]
        logging.info(f"Compiling proto file: {filepath}")

    subprocess.run(cmd)

def fix_proto_imports(filepath):
    logging.info(f"Fixing file at: {filepath}")
    cmd = [
        sys.executable,
        "-m",
        "protoletariat",
        "--create-package",
        "--in-place",
        "--python-out",
        package_name,
        "--module-suffixes",
        "_pb2.py",
        "--module-suffixes",
        "_pb2.pyi",
        "--module-suffixes",
        "_pb2_grpc.py",
        "--module-suffixes",
        "_pb2_grpc.pyi",
        "--module-suffixes",
        "_grpc.py",
        "--module-suffixes",
        "_grpc.pyi",
        "protoc",
        f"--proto-path={absolute_path}",
        filepath,
    ]
    subprocess.run(cmd)

def walk_through_project_and_compile_proto(directory):
    for root, dirs, files in os.walk(directory):
        for filename in files:
            if filename.endswith(".proto"):
                run_protoc(os.path.abspath(os.path.join(root, filename)))

def walk_through_project_and_fix_imports(directory):
    for root, dirs, files in os.walk(directory):
        for filename in files:
            if filename.endswith(".proto"):
                fix_proto_imports(os.path.abspath(os.path.join(root,
                                                               filename)))
                logging.info(f"Fixed imports for {filename}")

def remove_all_compiled_python_files(directory):
    for root, dirs, files in os.walk(directory):
        for filename in files:
            if filename.endswith(".py") or filename.endswith(".pyi"):
                logging.info(f"Deleting {os.path.join(root, filename)}")
                os.remove(os.path.join(root, filename))

def rename_any_proto_imports(directory):
    for root, dirs, files in os.walk(directory):
        for filename in files:
            with open(os.path.join(root, filename), "r+") as file:
                lines = file.readlines()

            if 'import "google/protobuf/any.proto";\n' in lines:
                for line in lines:
                    file.write(
                        re.sub(
                            r'^import "google/protobuf/any.proto";\n',
                            'import "google/protobuf/cosmos_any.proto";\n',
                            line,
                        ))

            if 'import "google/protobuf/any.proto";\n' in lines:
                for line in lines:
                    file.write(
                        re.sub(
                            r'^import "google/protobuf/any.proto";\n',
                            'import "google/protobuf/cosmos_any.proto";\n',
                            line,
                        ))

# rename_any_proto_imports(package_name)
remove_all_compiled_python_files(package_name)
walk_through_project_and_compile_proto(package_name)
walk_through_project_and_fix_imports(package_name)


import socket
from flask import Flask, render_template
from flask_sqlalchemy import SQLAlchemy

app = Flask(__name__)
app.config['SECRET_KEY'] = 'LongAndRandomSecretKey'
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///blog.db'
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False

db = SQLAlchemy(app)

@app.route('/')
def index():
    return render_template('index.html')

if __name__ == '__main__':
    my_host = "127.0.0.1"
    free_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    free_socket.bind((my_host, 0))
    free_socket.listen(5)
    free_port = free_socket.getsockname()[1]
    free_socket.close()

    login_manager = LoginManager()
    login_manager.login_view = 'users.login'
    login_manager.init_app(app)

    from models import User

    @login_manager.user_loader
    def load_user(id):
        return User.query.get(int(id))

    # blueprints
    from users.views import users_blueprint
    from blog.views import blog_blueprint

    app.register_blueprint(users_blueprint)
    app.register_blueprint(blog_blueprint)

    app.run(host=my_host, port=free_port, debug=True)


# Import the requests library to make HTTP requests to the API
import requests

# Define the API key to authenticate with the exchange rate API for currency conversion
API_KEY = "xxxxxxxxxxxxxxxxxxxxxx"

# Define the URL of the API endpoint for fetching currency symbols
SYMBOLS_URL = 'https://api.exchangerate.host/symbols'

# Define a function to find matching currencies based on user input
def find_matching_currencies(currency):
    # Initialize an empty list to store the matching currencies
    matching_currencies = []

    # Send a GET request to the symbols API and store the response in a variable
    response = requests.get(SYMBOLS_URL)

    # Check if the API request was successful (HTTP status code 200)
    if response.status_code == 200:
        # Parse the JSON response into a Python dictionary
        data = response.json()

        # Check if the 'success' field in the response is True
        if data.get('success', False):
            # Retrieve the 'symbols' dictionary from the response
            symbols = data.get('symbols', {})

            # Iterate over each symbol in the 'symbols' dictionary
            for symbol, details in symbols.items():
                # Convert the description to lowercase and check if the user input is contained in it
                description = details.get('description', '').lower()
                if currency.lower() in description:
                    # If a match is found, append a tuple with description and symbol to the list of matching currencies
                    matching_currencies.append((description, symbol))

    # Return the list of matching currencies
    return matching_currencies

# Define a function to get the exchange rate between USD and the target currency
def get_exchange_rate(base_currency, target_currency):
    # Construct the URL for the exchange rate API
    url = f"https://v6.exchangerate-api.com/v6/{API_KEY}/latest/{base_currency}"

    # Send a GET request to the API and store the response in a variable
    response = requests.get(url)

    # Parse the JSON response into a Python dictionary
    data = response.json()

    # Retrieve the exchange rate for the target currency from the 'conversion_rates' field in the response
    return data.get('conversion_rates', {}).get(target_currency)

# Define the main function to execute the program
def main():
    print("----- Currency Symbol Finder -----")
    # Take a part of currency name as input from the user
    currency_part = input('Please Enter a part of Your Currency Name to find its Symbol: ')

    # Call the find_matching_currencies function and store the result in a variable
    matching_currencies = find_matching_currencies(currency_part)

    print("Search Results:")
    # Print each matching currency on a new line
    for currency in matching_currencies:
        print(f"{currency[0]} ({currency[1]})")

    print("----- Currency Converter -----")
    # Ask the user to enter the desired currency symbol based on the search results
    target_currency = input("Now, Please check the search result and get your desired symbol to find your currency rate with respect to USD: ").upper()

    # Call the get_exchange_rate function and store the result in a variable
    exchange_rate = get_exchange_rate("USD", target_currency)

    # Check if an exchange rate was found and print the result, or print an error message if not found
    if exchange_rate:
        print(f"One USD = {exchange_rate} {target_currency}")
    else:
        print("Currency not found or error occurred.")

    print("Thanks For Using the Program!")

# Check if the script is run directly and not imported as a module, and if so, call the main function
if __name__ == "__main__":
    main()


import time
import copy
import numpy as np
import torch.multiprocessing as mp
import cv2
import matplotlib.pyplot as plt
from multiprocessing import Process, Value, Manager
import gym
from envs.visual_ur5_reacher.reacher_env import ReacherEnv
from senseact.utils import tf_set_seeds, NormalizedEnv
import argparse
import multiprocessing as mp

def make_env(setup='Visual_UR5',
             ip='129.128.159.210',
             seed=9,
             camera_id=0,
             image_width=160,
             image_height=120,
             target_type='stationary',
             image_history=3,
             joint_history=1,
             episode_length=4.0,
             dt=0.04):
    # state
    np.random.seed(seed)
    rand_state = np.random.get_state()
    # Create Visual UR5 Reacher environment
    env = ReacherEnv(
            setup=setup,
            host=ip,
            dof=5,
            camera_id=camera_id,
            image_width=image_width,
            image_height=image_height,
            channel_first=True,
            control_type="velocity",
            target_type=target_type,
            image_history=image_history,
            joint_history=joint_history,
            reset_type="zero",
            reward_type="dense",
            derivative_type="none",
            deriv_action_max=5,
            first_deriv_max=2,
            accel_max=1.4,
            speed_max=0.3,
            speedj_a=1.4,
            episode_length_time=episode_length,
            episode_length_step=None,
            actuation_sync_period=1,
            dt=dt,
            run_mode="multiprocess",
            rllab_box=False,
            movej_t=2.0,
            delay=0.0,
            random_state=rand_state
        )
    env = NormalizedEnv(env)
    env.start()
    return env

class UR5Wrapper():
    def __init__(self,
                 setup='Visual_UR5',
                 ip='129.128.159.210',
                 seed=9,
                 camera_id=0,
                 image_width=160,
                 image_height=120,
                 target_type='stationary',
                 image_history=3,
                 joint_history=1,
                 episode_length=4.0,
                 dt=0.04,
                 ignore_joint=False,
                 ):
        self.env = make_env(
                        setup,
                        ip,
                        seed,
                        camera_id,
                        image_width,
                        image_height,
                        target_type,
                        image_history,
                        joint_history,
                        episode_length,
                        dt,
                        )

        self.observation_space = self.env.observation_space['image']
        self.ignore_joint = ignore_joint
        if ignore_joint:
            self.state_space = gym.spaces.Box(low=0, high=1., shape=(0, ), dtype=np.float32)
            pass
        else:
            self.state_space = self.env.observation_space['joint']

        self.action_space = self.env.action_space

    def step(self, action):
        obs_dict, reward, done, _ = self.env.step(action)
        if self.ignore_joint:
            return obs_dict['image'], None, reward, done, _
        else:
            return obs_dict['image'], obs_dict['joint'], reward, done, _

    def reset(self):
        obs_dict = self.env.reset()
        if self.ignore_joint:
            return obs_dict['image'], None
        else:
            return obs_dict['image'], obs_dict['joint']

    def terminate(self):
        self.env.terminate()

if __name__ == '__main__':
    pass


import numpy as np

import pandas as pd

from scipy.spatial import distance
from scipy.stats import mode

import matplotlib.pyplot as plt

from sklearn import datasets
from sklearn.cluster import DBSCAN as sklearn_dbscan

class DBSCAN(object):
    def __init__(self, epsilon=3, min_points=3):
        self.epsilon = epsilon
        self.min_points = min_points

    def is_cluster_in_cluster(self, idx, cluster, clusters):
        res = []
        for i, c in enumerate(clusters):
            if (i==idx):
                continue
            if (len(np.intersect1d(cluster, c)) > 0):
                res.append(i)

        return res

    def expand_cluster(self, neighbors_per_point, point, visited):
        arr = np.array([])
        for i, p in enumerate(point):
            arr = np.union1d(arr, neighbors_per_point[p])
            visited[point[i]] = 1

        return arr

    def fit(self, X):
        X_distances = distance.cdist(X, X, 'euclidean')

        # init neighbors
        neighbors_per_point = []

        for data in X:
            neighbors_per_point.append([])

        # find neighbor in each point
        for i, distances in enumerate(X_distances):
            neighbors_per_point[i] = np.nonzero(distances <= self.epsilon)[0]

        neighbors_per_point = np.array(neighbors_per_point)

        clusters = []
        visited = np.zeros(X.shape[0])

        # creating clusters
        for i, point in enumerate(neighbors_per_point):
            if visited[i] == 1:
                continue
            if len(point) >= self.min_points:
                visited[i] = 1
                cluster = self.expand_cluster(neighbors_per_point, point, visited)
                idx_another_cluster = self.is_cluster_in_cluster(i, cluster, clusters)
                if (len(idx_another_cluster) > 0):
                    another_cluster = np.array([])
                    for idx in idx_another_cluster:
                        another_cluster = np.concatenate((another_cluster, clusters[idx]), axis=None)
                    for idx in reversed(idx_another_cluster):
                        del clusters[idx]
                    cluster = np.union1d(cluster, another_cluster)
                clusters.append(cluster)

        self.clusters = clusters
        return

    def replace_labels(self, pred_labels):
        dict_replace = {
            -1: 2,
            0: 0,
            1: 1
        }
        pred_labels = np.array([dict_replace[label] for label in pred_labels])

        return pred_labels

    def fit_predict(self, X):
        self.fit(X)
        label = np.full((len(X)), -1)
        for i, data in enumerate(X):
            if (len(self.clusters) > 0):
                for j, c in enumerate(self.clusters):
                    if (np.any(c == i)):
                        label[i] = j

        self.labels = self.replace_labels(label)
        return self.labels

    def accuracy_score(self, y):
        print('Accuracy score: {}'.format((len(y) - len(np.where(self.labels != y)[0])) / len(y)))
        return

def main():
    # import dataset
    iris = datasets.load_iris()
    X = iris.data
    y = iris.target

    # params
    epsilon = 0.5
    min_points = 14

    # ini model
    model = DBSCAN(epsilon=epsilon, min_points=min_points)

    # results
    y_pred = model.fit_predict(X)
    print('Self implementation DBSCAN')
    model.accuracy_score(y)

    # replacing result, adjusting with y
    dict_replace = {
        -1: 2,
        0: 0,
        1: 1
    }

    # comparing with sklearn's DBSCAN
    sklearn_model = sklearn_dbscan(eps=epsilon, min_samples=min_points)

    y_pred_sklearn = sklearn_model.fit_predict(X)
    y_pred_sklearn = [dict_replace[pred] for pred in y_pred_sklearn]

    print('scikit-learn\'s DBSCAN')
    print('Accuracy score: {}'.format((len(y) - len(np.where(y_pred_sklearn != y)[0])) / len(y)))

if __name__ == "__main__":
    main()


from django.urls import path
from . import views
urlpatterns = [
    path('', views.login_reg),
    path ('register', views.register),
    path ('login', views.Login),
    path ('books', views.all_books),
    path ('books/add', views.addBook),
    path ('books/create', views.createBook),
    path ('books/<int:id>', views.newBook),

]


import cv2
import handTrackingModule as htm
import math
import time
import pywifi
from pywifi import const

class WifiCtrl():
    def Wifi(self):
        #############################################################
        wifi = pywifi.PyWiFi()
        iface = wifi.interfaces()[0]

        iface.disconnect()
        time.sleep(1)
        assert iface.status() in \
               [const.IFACE_DISCONNECTED, const.IFACE_INACTIVE]

        wifiId = pywifi.Profile()
        wifiId.ssid = 'Ridhim'
        wifiId.akm.append(const.AKM_TYPE_WPA2PSK)
        wifiId.auth = const.AUTH_ALG_OPEN
        wifiId.key = 'abcde'
        wifiId.cipher = const.CIPHER_TYPE_CCMP

        temp = iface.add_network_profile(wifiId)
        #############################################################

        wCam, hCam = 640, 480
        cap = cv2.VideoCapture(0)
        cap.set(3, wCam)
        cap.set(4, hCam)
        pTime = 0

        detector = htm.handDetector(detectionCon=0.7)

        while True:
            success, img = cap.read()
            img = detector.findHands(img)
            lmList = detector.findPosition(img, draw=False)
            if len(lmList) != 0:
                # print(lmList[4], lmList[8])

                x1, y1 = lmList[4][1], lmList[4][2]
                x2, y2 = lmList[8][1], lmList[8][2]
                cv2.line(img, (x1, y1), (x2, y2), (255, 0, 255), 3)
                length1 = math.hypot(x2 - x1, y2 - y1)
                if length1 < 20:
                    iface.connect(temp)
                    time.sleep(1)
                    assert iface.status() == const.IFACE_CONNECTED

                x3, y3 = lmList[20][1], lmList[20][2]
                cv2.line(img, (x1, y1), (x3, y3), (255, 0, 255), 3)
                length2 = math.hypot(x3 - x1, y3 - y1)
                # print(length1)
                if length2 < 20:
                    break

                x4, y4 = lmList[16][1], lmList[16][2]
                cv2.line(img, (x1, y1), (x4, y4), (255, 0, 255), 3)
                length3 = math.hypot(x4 - x1, y4 - y1)
                if length3 < 20:
                    iface.disconnect()
                    time.sleep(1)
                    assert iface.status() in \
                           [const.IFACE_DISCONNECTED, const.IFACE_INACTIVE]

            cTime = time.time()
            fps = 1 / (cTime - pTime)
            pTime = cTime

            cv2.putText(img, f'FPS: {int(fps)}', (40, 50), cv2.FONT_HERSHEY_COMPLEX,
                        1, (255, 0, 0), 3)

            cv2.imshow("Img", img)
            cv2.waitKey(1)

        cap.release()
        cv2.destroyAllWindows()

def main():
    object = WifiCtrl()
    object.Wifi()

if __name__ == "__main__":
    main()


import asyncio
import aio_pika
import base64

async def main():
    connection = await aio_pika.connect_robust("amqp://guest:guest@localhost/")

    async with connection:
        channel = await connection.channel()
        await channel.default_exchange.publish(
            aio_pika.Message(body=base64.b64encode(open('image.png', 'rb').read())),
            routing_key='image_queue'
        )

if __name__ == "__main__":
    asyncio.run(main())


from typing import Dict

from dm_control import mjcf
from dm_control.composer.observation import observable
from dm_robotics.moma import sensor as moma_sensor
import numpy as np

class ExternalValueSensor(moma_sensor.Sensor):
  """Sensor to expose some externally defined value.

  This can be useful when we need to expose to the agent some value that is
  determined by a non-physical process (eg: the progress of a curriculum).
  """

  def __init__(self, name: str, initial_value: np.ndarray):
    self._name = name
    self._value = initial_value

    self._observables = {
        self._name: observable.Generic(lambda _: self._value),
    }

    for obs in self._observables.values():
      obs.enabled = True

  def initialize_episode(self, physics: mjcf.Physics,
                         random_state: np.random.RandomState) -> None:
    pass

  @property
  def observables(self) -> Dict[str, observable.Observable]:
    return self._observables

  @property
  def name(self) -> str:
    return self._name

  def get_obs_key(self, obs) -> str:
    del obs
    return self._name

  def set_value(self, value: np.ndarray) -> None:
    if value.shape != self._value.shape:
      raise ValueError('Incompatible value shape')
    self._value = value


import aiohttp_jinja2
from aiohttp import web
import os
from http_storage.files_processing import file_writer, unique_hash, get_dir, simple_hash, file_sender, file_remover
from http_storage.settings import config

HASH_TYPE = config['http_storage']['hash_type']

@aiohttp_jinja2.template('upload.html')
async def upload(request):
    pass

async def upload_handler(request) -> web.json_response:
    print(request.headers)
    reader = await request.multipart()
    field = await reader.next()
    assert field.name == 'file'
    filename = field.filename

    hashed_filename = unique_hash(filename) if HASH_TYPE else simple_hash(filename)

    await file_writer(hashed_filename, field)
    return web.json_response({'status': 'uploaded', 'file_id': hashed_filename})

async def download_handler(request) -> web.json_response:
    hashed_filename = request.query['file_id']
    file_store_path = os.path.join(get_dir(hashed_filename), hashed_filename)
    if os.path.exists(file_store_path):
        headers = {f'Content-disposition': f'attachment; filename={hashed_filename}'}
        return web.Response(body=file_sender(file_store_path=file_store_path), headers=headers, status=200)
    else:
        return web.json_response({'file_id': hashed_filename, 'status': 'file not found.'})

async def remove_handler(request) -> web.json_response:
    hashed_filename = request.query['file_id']
    try:
        await file_remover(hashed_filename)
        return web.json_response({'file_id': hashed_filename, 'status': 'file deleted.'})
    except FileNotFoundError:
        return web.json_response({'file_id': hashed_filename, 'status': 'file not found.'})


# %% Setup
import gradio as gr
import openai
openai.api_key = open("key.txt", "r").read().strip('\n')

# %% Give it a message history where you explain the task
message_history = [{'role':'user', 'content':f"You are a Shakespeare bot. I will provide you with a scenario or subjects. You must only reply with a Shakespearean sonnet about that scene or subjects. Reply with OK if you understand"}, {'role':'assistant', 'content':f'OK'}]

# %% Create the function to return responses
def predict(input):
    global message_history
    message_history.append({'role':'user', 'content':input})
    completion = openai.ChatCompletion.create(
        model="gpt-3.5-turbo",
        messages=message_history
    )
    reply_content = completion.choices[0].message.content
    print(reply_content)
    message_history.append({'role':'assistant','content':reply_content})

    # response is a set of tuples with input and replies,
    response = [(message_history[i]["content"], message_history[i+1]["content"]) for i in range(2, len(message_history)-1, 2)]

    return response

# %% Build the gradio app
with gr.Blocks() as demo:
    chatbot = gr.Chatbot()
    with gr.Row():
        txt = gr.Textbox(show_label=False, placeholder="Type message here").style(container=False)
        txt.submit(predict, txt, chatbot) # submitting makes bot uses predict to gen a response
        # txt.submit(lambda: "", None, txt) # does same thing as below line but slower
        txt.submit(None, None, txt, _js="() => {''}")   # empty the the box when submitted

demo.launch()
# %%


import json

with open('dados.json', 'r') as f:
    faturamento_diario = json.load(f)

valores_faturamento = [d.get('valor', 0) for d in faturamento_diario]

menor_faturamento = min(valores_faturamento)
maior_faturamento = max(valores_faturamento)

faturamento_sem_zero = [f for f in valores_faturamento if f != 0]

if len(faturamento_sem_zero) > 0:
    media_faturamento = sum(faturamento_sem_zero) / len(faturamento_sem_zero)
else:
    media_faturamento = 0

dias_acima_da_media = len([f for f in faturamento_sem_zero if f > media_faturamento])

print(f'Menor faturamento diário: R$ {menor_faturamento:.2f}')
print(f'Maior faturamento diário: R$ {maior_faturamento:.2f}')
print(f'Média de faturamento diário: R$ {media_faturamento:.2f}')
print(f'Número de dias acima da média mensal: {dias_acima_da_media}')


import json, re
from collections import namedtuple
from itertools import chain

from markupsafe import escape
from lark import Lark, Transformer, v_args
import vertex2tex
import mistletoe
from mistletoe.span_token import SpanToken
from mistletoe.latex_token import Math as MathToken
from mistletoe.html_renderer import HTMLRenderer

from pfsc import check_config
import pfsc.constants
from pfsc.excep import PfscExcep, PECode
from pfsc_util.scan import PfscModuleStringAwareScanner

def vertex_and_escape(s):
    s = vertex2tex.translate_document(s, keychar=pfsc.constants.VERTEX_KEY_CHAR)
    return escape(s)

##################################################################################
# JSON Parsing
#
# We use a custom json parser for several reasons:
#   * Allow mutli-line strings as values, i.e. strings where a linebreak in the
#     input does not terminate the string but is simply accepted as a \n char.
#   * Allow strings to be delimited by " or '. Whichever is used, it can occur
#     within the string if escaped (as \" or \'). No other escapes are
#     processed.
#   * Allow strings to be delimited by """ or '''. In such strings, no escapes
#     are processed. WYSIWYG.
#   * Allow an extra comma at the end of an array or object.
#   * Allow both Python (True, False, None) and Javascript (true, false, null)
#     constants.
#   * Allow identifiers (as well as strings) as object keys, like in Javascript.
#   * Allow libpaths as primitives (along with strings, ints, floats, booleans,
#     and null). This requires a "scope" (any PfscObj instance) as a place to
#     resolve the libpath to an object. If no scope is provided, we simply
#     convert the libpath directly into a `Libpath` instance (a subclass of
#     `str` -- see below). Otherwise we resolve to an object. If the resolution
#     fails, we raise an exception. If the object is a PfscAssignment, it is
#     replaced by its RHS. Otherwise we again convert to a `Libpath` instance.

# We use a `json_` prefix to put the grammar definition within a namespace, so
# that it can be included as a sub-grammar elsewhere.
json_grammar = r"""
    ?json_value: json_object
          | json_array
          | ve_string
          | SIGNED_INT         -> json_integer
          | SIGNED_FLOAT       -> json_number
          | ("true"|"True")    -> json_true
          | ("false"|"False")  -> json_false
          | ("null"|"None")    -> json_null
          | json_libpath
    json_array  : "[" [json_value ("," json_value)*] ","? "]"
    json_object : "{" [json_pair ("," json_pair)*] ","? "}"
    json_pair   : (json_cname|ve_string) ":" json_value
    ve_string : TRIPLE_QUOTE_STRING|TRIPLE_APOS_STRING|ESCAPED_STRING|APOS_STRING
    json_cname: CNAME
    json_libpath: CNAME ("." CNAME)*
"""

json_grammar_imports = """
    TRIPLE_APOS_STRING.2 : "'''" /'?'?[^']/* "'''"
    TRIPLE_QUOTE_STRING.2 : "\\"\\"\\"" /"?"?[^"]/* "\\"\\"\\""
    APOS_STRING : "'" ("\\'"|/[^']/)* "'"
    %import common.CNAME
    %import common.ESCAPED_STRING
    %import common.SIGNED_INT
    %import common.SIGNED_FLOAT
    %import common.WS
    %ignore WS
"""

class Libpath(str):
    """
    The purpose of this subclass of str is to make a formal record of the fact
    that a string returned after parsing our extended JSON syntax was
    originally given in the .pfsc module not as a literal string, but as a
    direct libpath reference.
    """

    def __new__(cls, value):
        self = str.__new__(cls, value)
        return self

    def resolve_to_rhs_or_not_at_all(self, scope):
        if scope is None:
            return self
        obj, ancpath = scope.getAsgnValueFromAncestor(self)
        if obj is not None:
            # We got the RHS of a PfscAssignment
            return obj
        obj, ancpath = scope.getFromAncestor(self)
        if obj is None:
            # This means we couldn't resolve the libpath at all, which is an exception.
            msg = 'Libpath %s could not be resolved.' % self
            raise PfscExcep(msg, PECode.RELATIVE_LIBPATH_CANNOT_BE_RESOLVED)
        # We can resolve the libpath, but it doesn't point to a PfscAssignment,
        # so we just return the libpath itself.
        return self

class PfscJsonTransformer(Transformer):

    def __init__(self, scope=None):
        """
        :param scope: a PfscObj where libpaths can be resolved.
        """
        self.scope = scope

    def json_libpath(self, items):
        libpath = Libpath('.'.join(items))
        return libpath.resolve_to_rhs_or_not_at_all(self.scope)

    @v_args(inline=True)
    def ve_string(self, s):
        """
        "VE-string" stands for "VerTeXed and Escaped string".
        This means that (1) VerTeX has been translated to ordinary TeX, and
        (2) HTML escaping has been applied (in that order).
        """
        # First we have to strip the delimiters, and replace escaped quotes
        # in single-quoted strings.
        lc = s[0]
        if lc == '"':
            n = 3 if s[:3] == '"""' else 1
        else:
            n = 3 if s[:3] == "'''" else 1
        s = s[n:-n]
        if n == 1:
            s = s.replace('\\'+lc, lc)
        return vertex_and_escape(s)

    json_array = list
    json_pair = tuple
    json_object = dict
    json_cname = v_args(inline=True)(str)
    json_integer = v_args(inline=True)(int)
    json_number = v_args(inline=True)(float)

    json_null = lambda self, _: None
    json_true = lambda self, _: True
    json_false = lambda self, _: False

json_parser = Lark(json_grammar + json_grammar_imports, start='json_value', parser='lalr', lexer='standard')

###############################################################################
# Widget parsing

# RawWidgetData is a simple named tuple for representing the raw data extracted from widget
# definitions in Proofscape-flavored Markdown.
#
# The fields in the RawWidgetData tuple are:
#   type :  the type given in the <type:name> part
#   name:  the name given in the <type:name> part
#   label: the text strictly between the square brackets
#   data:  the full JSON data, including the outside braces
#   lineno: the line number (within the given text) on which the widget defn starts
#
RawWidgetData = namedtuple("RawWidgetData", "type name label data lineno")

class WidgetDataScanner(PfscModuleStringAwareScanner):

    def __init__(self):
        super().__init__()
        self.brace_depth = 1
        self.data_part = None
        self.remainder = None
        self.num_newlines = None

    def state_0(self, c, i):
        if c == "{":
            self.brace_depth += 1
        elif c == "}":
            self.brace_depth -= 1
            if self.brace_depth == 0:
                self.data_part = "{" + self.code[:i+1]
                self.remainder = self.code[i+1:]
                self.num_newlines = self.code.count('\n')
                return self.BREAK, None
        return None, None

WIDGET_RE = re.compile(r'<( *\w+ *):( *[a-zA-Z]\w* *)?>\[([^]]*)\]{')

def split_on_widgets(text, supply_missing_names=True):
    """
    This function is for identifying the widget definitions in a Proofscape annotation.
    Annotations are written in "Proofscape-flavored Markdown", and a widget definition occurring
    in the midst of such will look like this:

        Blah blah, surrounding text <widget_type:widget_name>[label text for widget, *where markdown is okay too*]{
            "here": "you give the data",
            "that": [
                "defines", "the", "widget"
            ],
            "in": {
                "javascript": 1,
                "object": 2,
                "notation": 3
            }
        } and then some more surrounding text, blah blah blah.

    OR, it may begin merely with `<widget_type:>`, omitting the `widget_name`.

    In parsing, we use a depth counter to allow nested braces, and hence arbitrary
    JSON objects for the data part of each widget.

    However, in the name of simplicity we do not allow brackets within widget labels,
    so we do not do depth counting there.

    :param text: The full text of a Proofscape annotation.
    :param supply_missing_names: If True (the default), names are automatically generated and inserted
             for any widgets that lack them. If False, an exception is raised if any widget lacks a name.
    :return: The list of "parts." This is similar to what you would get from re.split, if you
             were able to split on widgets. Thus, the list will always be of odd length, beginning
             and ending with a string. The entries in the list alternate between strings, and
             RawWidgetData named tuples.
    """
    lineno = 1
    # We start by splitting on the `<widget_type:widget_name>[label_text]{` regex.
    chunks = WIDGET_RE.split(text)
    num_chunks = len(chunks)
    assert num_chunks % 4 == 1
    # Number of quadruples to be processed:
    Nquad = int((num_chunks - 1)/4)
    # Initialize return value.
    parts = [chunks[0]]
    lineno += chunks[0].count('\n')
    # Iterate over quadruples.
    names = []
    indices_of_missing_names = []
    for k in range(Nquad):
        a, b, c, remainder = chunks[4*k+1 : 4*k+5]
        widget_type = a.strip()
        widget_label = c.strip()
        widget_name = None
        widget_lineno = lineno
        if b is None:
            # Widget is unnamed.
            if not supply_missing_names:
                # If we're not meant to supply missing names, raise an exception.
                msg = 'Widget missing name: "<%s:>[%s]{..."' % (a, c)
                raise PfscExcep(msg, PECode.WIDGET_MISSING_NAME)
            else:
                # If we _are_ meant to supply missing names, note the index for later.
                indices_of_missing_names.append(k)
        else:
            # Widget is named.
            widget_name = b.strip()
            # Add to list of names.
            names.append(widget_name)
        wds = WidgetDataScanner()
        wds.scan(remainder)
        if wds.brace_depth != 0:
            msg = 'Unterminated widget: "<%s:>[%s]{..."' % (a, c)
            raise PfscExcep(msg)
        # Add the widget part.
        parts.append(RawWidgetData(widget_type, widget_name, widget_label,
                          wds.data_part, widget_lineno))
        # And the "remainder of the remainder" is the next non-widget part.
        parts.append(wds.remainder)
        lineno += wds.num_newlines

    # Do we need to supply missing names?
    if indices_of_missing_names:
        # Yes, there are names to be supplied.
        # All generated names will be of the form `w<n>` where n is a positive integer.
        # First we must determine which such names are already taken. And here we are
        # case-insensitive on the leading `w`.
        used_nums = set()
        for name in names:
            if len(name) >= 2 and name[0] in 'wW' and name[1] != '0':
                try:
                    n = int(name[1:])
                except ValueError:
                    pass
                else:
                    used_nums.add(n)
        def next_num(used):
            n = 1
            while True:
                while n in used: n += 1
                yield n
                n += 1
        for k, n in zip(indices_of_missing_names, next_num(used_nums)):
            rwd = parts[2*k+1]
            parts[2*k+1] = rwd._replace(name='w%s' % n)
    # Return the parts.
    return parts

###############################################################################
# Markdown processing

def render_markdown(text, trusted=False):
    """
    You can use this function to render markdown that doesn't necessarily come
    from an actual annotation, but using the process that is applied to annotation
    text. In this case, you pass no widget lookup since the text is not expected
    to contain any widget stubs. You do get to specify whether the "trusted" or
    "untrusted" policies should be applied while rendering.

    :param text: the markdown to be rendered
    :param trusted: boolean specifying how this text should be treated
    :return: rendered HTML
    """
    return render_anno_markdown(text, {}, trusted=trusted)

DOMAIN_LIST_PATTERN = re.compile(r'\w+\.\w+(\.\w+)*(, *\w+\.\w+(\.\w+)*)*$')

def process_domain_policy(policy):
    if policy in ['0', 0]:
        allow = False
    elif policy in ['1', 1]:
        allow = True
    elif not DOMAIN_LIST_PATTERN.match(policy):
        msg = f"Malformed policy: {policy}"
        raise PfscExcep(msg, PECode.MALFORMED_DOMAIN_POLICY)
    else:
        allow = [d.strip() for d in policy.split(',')]
    return allow

def lookup_link_and_img_policy(trusted):
    repo_type = "TRUSTED" if trusted else "UNTRUSTED"
    link_policy = check_config(f"PFSC_MD_LINKS_FOR_{repo_type}_REPOS")
    img_policy = check_config(f"PFSC_MD_IMGS_FOR_{repo_type}_REPOS")
    allow_links = process_domain_policy(link_policy)
    allow_images = process_domain_policy(img_policy)
    return allow_links, allow_images

def render_anno_markdown(annotext_with_widget_stubs, widget_lookup, trusted=False):
    """
    IMPORTANT: This is THE ONLY function you should use when it is time to turn
      annotation markdown into HTML.

    :param annotext_with_widget_stubs: (string) the annotation text after replacing
      widgets with widget stubs
    :param widget_lookup: (dict) the lookup where we can find the actual Widget
      instance for each stub
    :param trusted: boolean saying whether this text is being treated as coming
      from a trusted source or not. This controls (in combination with the app
      config) how links and images in the markdown will be treated.
    :return: the rendered HTML
    """
    allow_links, allow_images = lookup_link_and_img_policy(trusted)
    renderer = PfscRenderer(widget_lookup, allow_links=allow_links, allow_images=allow_images)
    ve_text = vertex_and_escape(annotext_with_widget_stubs)
    return mistletoe.markdown(ve_text, renderer)

MathToken.precedence = 100

class PfscWidgetStub(SpanToken):
    """
    A widget stub represents a widget with a small, inline element that just gives
    the label text of the widget, and the widget's name, in the form `[label]{name}`.

    This allows us to (1) apply ordinary Markdown rendering recursively on the label part,
    (2) look up a widget based on its name, and (3) pass the rendered label HTML to the
    widget, for it to compute the final HTML that will replace the stub.

    The [Mistletoe documentation](https://github.com/miyuchina/mistletoe#a-new-token)
    explains how to extend the lanaguage by defining a new span token, as we've done here.
    """
    # Widget stub format is `[label]{name}`:
    pattern = re.compile(r"\[([^]]*)\]{(\w+)}")
    # Apply markdown processing recursively to the _first_ matching group, i.e. the
    # label in `[label]{name}`:
    parse_inner = True
    parse_group = 1
    # Set high precedence, just under that of MathTokens.
    precedence = 90
    def __init__(self, match_obj):
        super().__init__(match_obj)
        self.widget_name = match_obj.group(2)

class SectionNumberRenderer(HTMLRenderer):
    """
    Automatically add section numbers to headings.

        self.sn_do_number: default False; set True to add section numbers.
        self.sn_top_level: default 1; start numbering on headings of
          this level; must be int from 1 to 6.
    """

    def __init__(self, *extras):
        super().__init__(*extras)
        self.sn_counters = [0] * 6
        self.sn_do_number = False
        self.sn_top_level = 1

    def render_heading(self, token):
        template = '<h{level}>{inner}</h{level}>'
        inner = self.render_inner(token)
        level = token.level
        self.sn_counters[level - 1] += 1
        for i in range(level, 6):
            self.sn_counters[i] = 0
        if self.sn_do_number and level >= self.sn_top_level:
            N = self.sn_counters[self.sn_top_level - 1 : level]
            number = '.'.join(map(str, N))
            inner = f'{number}   {inner}'
        return template.format(level=level, inner=inner)

class MathRenderer(SectionNumberRenderer):
    """
    This is our base HTML renderer class. It correctly identifies TeX math modes,
    giving these higher precedence than anything else in Markdown. This is necessary,
    for example, to avoid applying  tags in

        foo $a_1, a_2$ bar

    around the text between the two underscore characters in the math mode, as in

        foo $a1, a2$ bar
    """

    def __init__(self, *extras):
        super().__init__(*chain((MathToken,), extras))

    @staticmethod
    def render_math(token):
        return token.content

class PfscRenderer(MathRenderer):
    """
    This class does rendering for Proofscape-flavored Markdown.
    Its extensions over the base HTMLRenderer are as follows:

    (1) Correctly identify TeX math modes, giving these higher precedence
    than anything else in Markdown. This is necessary, for example, to avoid
    applying  tags in

        foo $a_1, a_2$ bar

    around the text between the two underscore characters in the math mode, as in

        foo $a1, a2$ bar

    (2) Identify widget stubs, and replace these with the HTML for the named
    widget. The latter is obtained from a lookup, passed to our constructor.

    (3) Mark links with class "external" and target "_blank".

    (4) Accept configuration parameters saying whether links are allowed,
    and whether images are allowed.

    (5) Blocks all HTML pass-through. This is intended only as a backup, since
    we expect that this renderer will only ever be applied to markdown that has
    already been HTML escaped.
     Notes:
      * [HTML pass-through is part of the CommonMark spec](https://spec.commonmark.org/0.28/#html-blocks)
      * [Some wish it weren't](https://talk.commonmark.org/t/remove-html-passthru-pass-through/1869)

    (6) Since we anticipate receiving only HTML-escaped text, this text will not
    include any angle brackets. Therefore it is impossible for the user to use
    `<URL>` style links. To make up for this lack, we add a special rule (outside
    the MD spec): if you use a link of the form `[](URL)`, i.e. with empty label,
    then we automatically set the URL as the label.
    """

    def __init__(self, widget_lookup, allow_links=False, allow_images=False):
        """
        :param widget_lookup: dict mapping widget names to Widget objects
        :param allow_links: boolean (meaning all or nothing) or list of allowed domains
        :param allow_images: boolean (meaning all or nothing) or list of allowed domains
        """
        super().__init__(PfscWidgetStub)
        self.widget_lookup = widget_lookup
        self.allow_links = allow_links
        self.allow_images = allow_images

    def __call__(self, *args, **kwargs):
        """
        This allows us to pass an _instance_ of this class -- rather than the
        class itself -- to `mistletoe.markdown`.
        """
        return self

    def render_pfsc_widget_stub(self, token):
        from pfsc.lang.widgets import CtlWidget
        widget_name = token.widget_name
        widget = self.widget_lookup.get(widget_name)
        if widget:
            if isinstance(widget, CtlWidget):
                widget.configure(self)
                return ''
            else:
                inner = self.render_inner(token)
                return widget.writeHTML(label=inner)
        else:
            return f'MISSING_WIDGET:[...]{{{widget_name}}}'

    @staticmethod
    def make_link_external(a_tag):
        i0 = a_tag.find('<a') + 2
        e = ' target="_blank" class="external"'
        a_tag = a_tag[:i0] + e + a_tag[i0:]
        return a_tag

    def render_inline_code(self, token):
        """
        For some reason (maybe just an oversight), Mistletoe's `HTMLRenderer` base
        class uses straight `html.escape` here, like this:
            inner = html.escape(token.children[0].content)
        instead of using its `self.escape_html`. The purpose of the latter is to prevent
        double-escaping. For us, that case definitely arises, since we pre-escape code.
        So we have to override the base class method just to use the right `self.escape_html` here.
        """
        template = '{}'
        inner = self.escape_html(token.children[0].content)
        return template.format(inner)

    def render_block_code(self, token):
        """
        Our reason for overriding the base class's `render_block_code` is exactly the
        same as our reason for overriding `render_inline_code` -- see above. We just need
        to apply `self.escape_html` instead of `html.escape` to the inner content.
        """
        template = '
{inner}
'
        if token.language:
            attr = ' class="{}"'.format('language-{}'.format(self.escape_html(token.language)))
        else:
            attr = ''
        inner = self.escape_html(token.children[0].content)
        return template.format(attr=attr, inner=inner)

    # -----------------------------------------------------------------------
    # Block HTML pass-through

    def render_html_span(self, token):
        h = super().render_html_span(token)
        h = self.escape_html(h)
        return h

    def render_html_block(self, token):
        h = super().render_html_block(token)
        h = self.escape_html(h)
        return h

    # -----------------------------------------------------------------------
    # Enforce the configured policy.

    @staticmethod
    def url_is_okay(url, allow):
        """
        To be "okay" a URL not only has to match our policy, but also
        must use an accepted scheme, namely `http` or `https`.
        :param url: the URL to be checked
        :param allow: boolean (meaning all domains or none) or list of domain names
        :return: boolean
        """
        from pfsc.checkinput import check_url
        if allow is False:
            return False
        params = {
            'allowed_schemes': ['http', 'https'],
        }
        if isinstance(allow, list):
            params['allowed_netlocs'] = allow
        else:
            assert allow is True
        try:
            checked = check_url('', url, params)
        except PfscExcep as pe:
            if pe.code() == PECode.BAD_URL:
                return False
            else:
                raise pe
        if checked.scheme_ok and (checked.netloc_ok is True or allow is True):
            return True
        return False

    def render_image(self, token):
        h = super().render_image(token)
        url = token.src
        if not self.url_is_okay(url, self.allow_images):
            h = self.escape_html(h)
        return h

    def super_render_link_plus(self, token):
        """
        This is almost an exact copy of the `render_link` method from
        our superclass (hence "super_render_link"), plus a bit more
        (hence "_plus"). The bit more is where we implement our special
        rule that in links of the form `[](url)` i.e. with empty label,
        we automatically set the URL as the label text.
        """
        template = '{inner}'
        target = self.escape_url(token.target)
        if token.title:
            title = ' title="{}"'.format(self.escape_html(token.title))
        else:
            title = ''
        inner = self.render_inner(token)
        if inner == '':
            inner = target
        return template.format(target=target, title=title, inner=inner)

    def render_link(self, token):
        h = self.super_render_link_plus(token)
        h = self.make_link_external(h)
        url = token.target
        if not self.url_is_okay(url, self.allow_links):
            h = self.escape_html(h)
        return h

    def render_auto_link(self, token):
        h = super().render_auto_link(token)
        h = self.make_link_external(h)
        url = token.target
        if not self.url_is_okay(url, self.allow_links):
            h = self.escape_html(h)
        return h


import awkward as ak
import numpy as np
import coffea.processor as processor
from coffea.util import save

from coffea.nanoevents.methods import candidate
ak.behavior.update(candidate.behavior)

import random

from tools.collections import *
from tools.utils import *

class EventSelectorProcessor(processor.ProcessorABC):
    def __init__(self, analyzer_name, year):
        self.analyzer_name = analyzer_name
        self.year = year

        self._accumulator = processor.dict_accumulator({
            'cutflow': processor.defaultdict_accumulator(int),
        })

    @property
    def accumulator(self):
        return self._accumulator

    def process(self, events):
        output = self.accumulator.identity()

        # test if there are any events in the file
        if len(events) == 0:
            return output

        ############### Get All the interesting candidates from NTuples
        Dimu = ak.zip({**get_vars_dict(events, dimu_cols)}, with_name="PtEtaPhiMCandidate")
        Muon = ak.zip({**get_vars_dict(events, muon_cols)}, with_name="PtEtaPhiMCandidate")
        D0 = ak.zip({'mass': events.D0_mass12, **get_vars_dict(events, d0_cols)}, with_name="PtEtaPhiMCandidate")
        Dstar = ak.zip({'mass': (events.Dstar_D0mass + events.Dstar_deltamr),
                        'charge': events.Dstar_pischg,
                        **get_vars_dict(events, dstar_cols)},
                        with_name="PtEtaPhiMCandidate")
        PVtx = ak.zip({**get_vars_dict(events, pvtx_cols)})
        HLT = ak.zip({**get_hlt(events, hlt_cols[self.year])})

        output['cutflow']['Number of events']  += len(events)
        output['cutflow']['Number of Dimu']    += ak.sum(ak.num(Dimu))
        output['cutflow']['Number of D0']      += ak.sum(ak.num(D0))
        output['cutflow']['Number of Dstar']   += ak.sum(ak.num(Dstar))

        ############### Dimu cuts charge = 0, mass cuts and chi2...
        Dimu = ak.mask(Dimu, Dimu.charge == 0)
        output['cutflow']['Dimu 0 charge'] += ak.sum(ak.num(Dimu[remove_none(Dimu.pt)]))

        Dimu = ak.mask(Dimu, ((Dimu.mass > 8.5) & (Dimu.mass < 11.5)) | ((Dimu.mass > 2.95) & (Dimu.mass < 3.25)))
        output['cutflow']['Quarkonia mass'] += ak.sum(ak.num(Dimu[remove_none(Dimu.pt)]))

        ############### Get the Muons from Dimu, for cuts in their params
        Muon = ak.zip({'0': Muon[Dimu.t1muIdx], '1': Muon[Dimu.t2muIdx]})

        # SoftId and Global Muon cuts
        soft_id = (Muon.slot0.softId > 0) & (Muon.slot1.softId > 0)
        Dimu = ak.mask(Dimu, soft_id)
        Muon = ak.mask(Muon, soft_id)
        output['cutflow']['Dimu muon softId'] += ak.sum(ak.num(Dimu[remove_none(Dimu.pt)]))

        """ global_muon = (Muon.slot0.isGlobal > 0) & (Muon.slot1.isGlobal > 0)
        Dimu = Dimu[global_muon]
        Muon = Muon[global_muon]
        output['cutflow']['Dimu muon global'] += ak.sum(ak.num(Dimu)) """

        # pt and eta cuts
        muon_pt_cut = (Muon.slot0.pt > 3) & (Muon.slot1.pt > 3)
        Dimu = ak.mask(Dimu, muon_pt_cut)
        Muon = ak.mask(Muon, muon_pt_cut)
        output['cutflow']['Dimu muon pt cut'] += ak.sum(ak.num(Dimu[remove_none(Dimu.pt)]))

        muon_eta_cut = (np.absolute(Muon.slot0.eta) < 2.4) & (np.absolute(Muon.slot1.eta) < 2.4)
        Dimu = ak.mask(Dimu, muon_eta_cut)
        Muon = ak.mask(Muon, muon_eta_cut)
        output['cutflow']['Dimu muon eta cut'] += ak.sum(ak.num(Dimu[remove_none(Dimu.pt)]))

        Dimu['is_ups'] = (Dimu.mass > 8.5) & (Dimu.mass < 11.5)
        Dimu['is_jpsi'] = (Dimu.mass > 2.95) & (Dimu.mass < 3.25)

        ############### Cuts for Dstar

        # trks cuts
        Dstar = Dstar[~Dstar.hasMuon]
        output['cutflow']['Dstar trk muon cut'] += ak.sum(ak.num(Dstar))

        Dstar = Dstar[(Dstar.Kpt > 0.5) & (Dstar.pipt > 0.5)]
        output['cutflow']['Dstar trk pt cut'] += ak.sum(ak.num(Dstar))

        Dstar = Dstar[(Dstar.Kchindof < 2.5) & (Dstar.pichindof < 2.5)]
        output['cutflow']['Dstar trk pt cut'] += ak.sum(ak.num(Dstar))

        Dstar = Dstar[(Dstar.KnValid > 4) & (Dstar.pinValid > 4) & (Dstar.KnPix > 1) & (Dstar.pinPix > 1)]
        output['cutflow']['Dstar trk hits cut'] += ak.sum(ak.num(Dstar))

        #Dstar = Dstar[(Dstar.Kdxy < 0.5/np.cos(2 * np.arctan(np.exp(-Dstar.Keta)))) & (Dstar.pidxy < 0.5/np.cos(2 * np.arctan(np.exp(-Dstar.pieta))))]
        #output['cutflow']['Dstar trk pt cut'] += ak.sum(ak.num(Dstar))

        #Dstar = Dstar[(Dstar.Kdz < 0.5/np.cos(2 * np.arctan(np.exp(-Dstar.Keta)))) & (Dstar.pidz < 0.5/np.cos(2 * np.arctan(np.exp(-Dstar.pieta))))]
        #output['cutflow']['Dstar trk pt cut'] += ak.sum(ak.num(Dstar))

        # pis cuts
        Dstar = Dstar[Dstar.pisptr > 0.3]
        output['cutflow']['Dstar pis pt cut'] += ak.sum(ak.num(Dstar))

        Dstar = Dstar[Dstar.pischir < 3]
        output['cutflow']['Dstar pis chi2 cut'] += ak.sum(ak.num(Dstar))

        Dstar = Dstar[Dstar.pisnValid > 2]
        output['cutflow']['Dstar pis hits cut'] += ak.sum(ak.num(Dstar))

        # D0 of Dstar cutsa
        Dstar = Dstar[Dstar.D0cosphi > 0.95]
        output['cutflow']['Dstar D0 cosphi cut'] += ak.sum(ak.num(Dstar))

        Dstar = Dstar[(Dstar.D0mass < D0_PDG_MASS + 0.040) & (Dstar.D0mass > D0_PDG_MASS - 0.040)]
        output['cutflow']['Dstar D0 mass cut'] += ak.sum(ak.num(Dstar))

        #Dstar = Dstar[Dstar.D0pt > 3]
        #output['cutflow']['Dstar D0 pt cut'] += ak.sum(ak.num(Dstar))

        Dstar = Dstar[Dstar.D0dlSig > 1]
        output['cutflow']['Dstar D0 dlSig cut'] += ak.sum(ak.num(Dstar))

        Dstar['wrg_chg'] = (Dstar.Kchg == Dstar.pichg)

        ############### Dimu + OpenCharm associations
        DimuDstar = association(Dimu, Dstar)
        DimuDstar = DimuDstar[ak.fill_none(DimuDstar.slot0.pt, -1) > -1]
        Dimu = Dimu[remove_none(Dimu.pt)]

        ############### Cuts for D0
        D0 = D0[~D0.hasMuon]
        output['cutflow']['D0 trk muon cut'] += ak.sum(ak.num(D0))

        D0 = D0[(D0.t1pt > 0.8) & (D0.t2pt > 0.8)]
        output['cutflow']['D0 trk pt cut'] += ak.sum(ak.num(D0))

        D0 = D0[(D0.t1chindof < 2.5) & (D0.t2chindof < 2.5)]
        output['cutflow']['D0 trk chi2 cut'] += ak.sum(ak.num(D0))

        D0 = D0[(D0.t1nValid > 4) & (D0.t2nValid > 4) & (D0.t1nPix > 1) & (D0.t2nPix > 1)]
        output['cutflow']['D0 trk hits cut'] += ak.sum(ak.num(D0))

        #D0 = D0[(D0.t1dxy < 0.5/np.cos(2 * np.arctan(np.exp(-D0.t1eta)))) & (D0.t2dxy < 0.5/np.cos(2 * np.arctan(np.exp(-D0.t2eta))))]
        #output['cutflow']['D0 trk dxy cut'] += ak.sum(ak.num(D0))

        #D0 = D0[(D0.t1dz < 0.5/np.cos(2 * np.arctan(np.exp(-D0.t1eta)))) & (D0.t2dz < 0.5/np.cos(2 * np.arctan(np.exp(-D0.t2eta))))]
        #output['cutflow']['D0 trk dz cut'] += ak.sum(ak.num(D0))

        # D0 cosphi
        D0 = D0[D0.cosphi > 0.99]
        output['cutflow']['D0 cosphi cut'] += ak.sum(ak.num(D0))

        # D0 dl Significance
        D0 = D0[D0.dlSig > 5.]
        output['cutflow']['D0 dlSig cut'] += ak.sum(ak.num(D0))

        # D0 pt
        D0 = D0[D0.pt > 3.]
        output['cutflow']['D0 pt cut'] += ak.sum(ak.num(D0))

        ############### Final computation of number of objects
        output['cutflow']['Dimu final']    += ak.sum(ak.num(Dimu))
        output['cutflow']['D0 final']      += ak.sum(ak.num(D0))
        output['cutflow']['Dstar final']   += ak.sum(ak.num(Dstar))
        output['cutflow']['Dimu Dstar Associated'] += ak.sum(ak.num(DimuDstar))

        ############### Leading and Trailing muon separation
        leading_mu = (Muon.slot0.pt > Muon.slot1.pt)
        Muon_lead = ak.where(leading_mu, Muon.slot0, Muon.slot1)
        Muon_trail = ak.where(~leading_mu, Muon.slot0, Muon.slot1)
        Muon_lead = Muon_lead[remove_none(Muon_lead.pt)]
        Muon_trail = Muon_trail[remove_none(Muon_trail.pt)]

        ############### Create the accumulators to save output
        muon_lead_acc = processor.dict_accumulator({})
        for var in Muon_lead.fields:
            muon_lead_acc[var] = processor.column_accumulator(ak.to_numpy(ak.flatten(Muon_lead[var])))
        muon_lead_acc["nMuon"] = processor.column_accumulator(ak.to_numpy(ak.num(Muon_lead)))
        output["Muon_lead"] = muon_lead_acc

        muon_trail_acc = processor.dict_accumulator({})
        for var in Muon_trail.fields:
            muon_trail_acc[var] = processor.column_accumulator(ak.to_numpy(ak.flatten(Muon_trail[var])))
        muon_trail_acc["nMuon"] = processor.column_accumulator(ak.to_numpy(ak.num(Muon_trail)))
        output["Muon_trail"] = muon_trail_acc

        dimu_acc = processor.dict_accumulator({})
        for var in Dimu.fields:
            if (var.startswith('t')): continue
            dimu_acc[var] = processor.column_accumulator(ak.to_numpy(ak.flatten(Dimu[var])))
        dimu_acc["nDimu"] = processor.column_accumulator(ak.to_numpy(ak.num(Dimu)))
        output["Dimu"] = dimu_acc

        D0_acc = processor.dict_accumulator({})
        D0_trk_acc = processor.dict_accumulator({})
        for var in D0.fields:
            if (var.startswith('t')):
                D0_trk_acc[var] = processor.column_accumulator(ak.to_numpy(ak.flatten(D0[var])))
            else:
                D0_acc[var] = processor.column_accumulator(ak.to_numpy(ak.flatten(D0[var])))
        D0_acc["nD0"] = processor.column_accumulator(ak.to_numpy(ak.num(D0)))
        output["D0"] = D0_acc
        output["D0_trk"] = D0_trk_acc

        Dstar_acc = processor.dict_accumulator({})
        Dstar_D0_acc = processor.dict_accumulator({})
        Dstar_trk_acc = processor.dict_accumulator({})
        for var in Dstar.fields:
            if var.startswith('D0'):
                Dstar_D0_acc[var] = processor.column_accumulator(ak.to_numpy(ak.flatten(Dstar[var])))
            elif (var.startswith('K') or var.startswith('pi')):
                Dstar_trk_acc[var] = processor.column_accumulator(ak.to_numpy(ak.flatten(Dstar[var])))
            else:
                Dstar_acc[var] = processor.column_accumulator(ak.to_numpy(ak.flatten(Dstar[var])))
        Dstar_acc["nDstar"] = processor.column_accumulator(ak.to_numpy(ak.num(Dstar)))
        output["Dstar"] = Dstar_acc
        output["Dstar_D0"] = Dstar_D0_acc
        output["Dstar_trk"] = Dstar_trk_acc

        DimuDstar_acc = processor.dict_accumulator({})
        DimuDstar_acc['Dimu'] = processor.dict_accumulator({})
        DimuDstar_acc['Dstar'] = processor.dict_accumulator({})
        for var in DimuDstar.fields:
            if (var == '0') or (var =='1'):
                continue
            elif var == 'cand':
                for i0 in DimuDstar[var].fields:
                    DimuDstar_acc[i0] = processor.column_accumulator(ak.to_numpy(ak.flatten(DimuDstar[var][i0])))
            else:
                DimuDstar_acc[var] = processor.column_accumulator(ak.to_numpy(ak.flatten(DimuDstar[var])))

        for var in DimuDstar.slot0.fields:
            DimuDstar_acc['Dimu'][var] = processor.column_accumulator(ak.to_numpy(ak.flatten(DimuDstar.slot0[var])))

        for var in DimuDstar.slot1.fields:
            DimuDstar_acc['Dstar'][var] = processor.column_accumulator(ak.to_numpy(ak.flatten(DimuDstar.slot1[var])))
        DimuDstar_acc['nDimuDstar'] = processor.column_accumulator(ak.to_numpy(ak.num(DimuDstar)))
        output['DimuDstar'] = DimuDstar_acc

        evt_info_acc = processor.dict_accumulator({})
        evt_info_acc['event'] = processor.column_accumulator(ak.to_numpy(events.event))
        evt_info_acc['run'] = processor.column_accumulator(ak.to_numpy(events.run))
        evt_info_acc['luminosityBlock'] = processor.column_accumulator(ak.to_numpy(events.luminosityBlock))
        output['event_info'] = evt_info_acc

        PVtx_acc = processor.dict_accumulator({})
        for var in PVtx.fields:
            PVtx_acc[var] = processor.column_accumulator(ak.to_numpy(ak.flatten(PVtx[var])))
        PVtx_acc['nPVtx'] = processor.column_accumulator(ak.to_numpy(ak.num(PVtx)))
        output['PVtx'] = PVtx_acc

        triggers_acc = processor.dict_accumulator({})
        for var in HLT.fields:
            triggers_acc[var] = processor.column_accumulator(ak.to_numpy(HLT[var]))
        output['triggers'] = triggers_acc

        file_hash = str(random.getrandbits(128)) + str(len(events))
        save(output, "output/" + self.analyzer_name + "/" + self.analyzer_name + "_" + file_hash + ".coffea")

        # return dummy accumulator
        return processor.dict_accumulator({
                'cutflow': output['cutflow']
        })

    def postprocess(self, accumulator):
        return accumulator


from __future__ import print_function

import matplotlib.pyplot as plt
import numpy as np
import pysparcl

from mpl_toolkits.axes_grid1 import make_axes_locatable
from scipy.cluster.hierarchy import dendrogram
from scipy.cluster.hierarchy import linkage
from scipy.spatial.distance import pdist
from scipy.spatial.distance import squareform

def show_dendrogram(dist):
    main_axes = plt.gca()
    divider = make_axes_locatable(main_axes)
    plt.sca(divider.append_axes("top", 1.5, pad=0))

    link = linkage(dist, method='average')
    dendro = dendrogram(
        link, no_labels=True, link_color_func=lambda x: 'black')

    plt.gca().set_axis_off()
    plt.gca().get_xaxis().set_visible(False)
    plt.gca().get_yaxis().set_visible(False)
    plt.sca(main_axes)

    distmat = squareform(dist)
    distmat /= distmat.max()
    indices = np.array(dendro['leaves'])
    distmat = (distmat[:, indices])[indices, :]

    plt.imshow(distmat, cmap='Blues', interpolation='nearest')
    plt.show()

if __name__ == '__main__':
    N = 50
    N_dim = 1000
    N_nonzero = 10
    np.random.seed(seed=1)

    class1 = np.zeros(N_dim)
    class2 = np.zeros(N_dim)
    perm = np.random.permutation(N_dim)[:N_nonzero]
    class1[perm] = 1
    class2[perm] = -1

    data = np.vstack(((np.dot(np.ones((N, 1)), [class1]),
                       np.dot(np.ones((N, 1)), [class2]))))
    data += np.random.randn(*data.shape)

    print('Perform hierarchical clustering...')
    dist = pdist(data, 'sqeuclidean')
    show_dendrogram(dist)

    print('Selecting tuning parameter for sparse hierarchical clustering...')
    perm = pysparcl.hierarchy.permute(data, verbose=True)

    print('Perform sparse hierarchical clustering...')
    result = pysparcl.hierarchy.pdist(data, wbound=perm['bestw'])
    show_dendrogram(result['u'])


import numpy as np
import json
import boto3
import os
import argparse
import pandas as pd
"""
Method for creating a train-test split of the availaible LISA images.

Input:
--------

bucket:
    The name of the bucket where the image dictionary is stored.

output:
    csv of of image dict keys for train and test, saved back to the bucket.
"""

class Make_Split(object):
    def __init__(self,bucket):
        self.s3 = boto3.resource("s3")
        self.bucket = bucket
        self.df = None
        self.signs_count_df = None
        self.top = None
        self.s3.meta.client.download_file(self.bucket, 'cropped_image_dict.json', 'cropped_image_dict.json')

        with open('cropped_image_dict.json') as data:
            self.image_dict = json.load(data)

        self.cropped_images = []
        for item in self.image_dict.items():
            if 'cropped' in item[1].keys():
                self.cropped_images.append([item[1]['type'],item[1]['cropped']])
        self.cropped_images = np.array(self.cropped_images)

    def build_dataframe(self):
        index = ['Row'+str(i) for i in range(1, len(self.cropped_images)+1)]
        self.df = pd.DataFrame(self.cropped_images, index=index)
        self.signs_count_df = self.df.groupby(0).agg('count')
        self.top = self.signs_count_df.sort_values(1, ascending=False).index[:15]
        self.df['gt_100'] = self.df[0].map(lambda x: self.highest_counts(x))
        self.df = self.df[self.df['gt_100'] == True].drop('gt_100', axis=1)

    def highest_counts(self, item):
        if item in self.top:
            return True
        else:
            return False

    def get_splits(self):
        first = 0
        for sign in self.df[0].unique():
            sign_type = self.df[self.df[0] == sign].values
            np.random.shuffle(sign_type)
            if first == 0:
                train, test = sign_type[:int(.8*len(sign_type))], sign_type[int(.8*len(sign_type)):]
                first += 1
            else:
                train = np.concatenate((train, sign_type[:int(.8*len(sign_type))]), axis = 0)
                test = np.concatenate((test, sign_type[int(.8*len(sign_type)):]), axis = 0)
        np.random.shuffle(train)
        np.random.shuffle(test)
        full = self.df.values
        np.random.shuffle(full)
        np.save('full',full)
        np.save('train',train)
        np.save('test',test)
        self.s3.meta.client.upload_file('full.npy',self.bucket,'full.npy')
        self.s3.meta.client.upload_file('train.npy',self.bucket,'train.npy')
        self.s3.meta.client.upload_file('test.npy',self.bucket,'test.npy')
        os.remove('full.npy')
        os.remove('train.npy')
        os.remove('test.npy')

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description='Method for parsing LISA sign data into cropped images.')
    parser.add_argument('--bucket', help='Input bucket name with cropped images')
    args = parser.parse_args()
    split_maker = Make_Split(args.bucket)
    split_maker.build_dataframe()
    split_maker.get_splits()