# THIS MODULE CONTAINS VARIOUS UTILITY FUNCTIONS USED
# BY ALL OTHER MODULES IN JETFREQ

import re
import jfexceptions
import os
from jfanalyze import sort_events
from jfanalyze import EventFreq
from jfanalyze import EventDiff
from jfanalyze import DiffType
from datetime import datetime

# THIS FUNCTION FORMATS AND WRITES DEBUG INFORMATION TO STDOUT
def debug(verbose, message):
	if verbose:
		if type(message) == list:
			for line in message:
				print("jetfreq.py: {}".format(line))
		else:
			print("jetfreq.py: {}".format(message))

# THIS FUNCTION TRUNCATES ANY PATH WITH A HIERARCHY GREATER THAN
# 4 DOWN TO THE FIRST 3 AND THE LAST 1 FOR BREVITY
def truncate_path(path, path_type):
	depth = 4 if path_type == 'dir' else 6;
	if '\\' in path:
		path_ary = path.split('\\')
		if len(path_ary) > depth:
			path = '\\'.join(path_ary[0:depth - 1]) + '\\ ... \\' + path_ary[len(path_ary) - 1]
	elif '/' in path:
		path_ary = path.split('/')
		if len(path_ary) > depth:
			path = '/'.join(path_ary[0:depth - 1]) + '/ ... /' + path_ary[len(path_ary) - 1]
	return path

# THIS FUNCTION SORT A LIST OF EVENT_DIFF OBJECTS
# ALPHABETICALLY BY THEIR DIFFTYPE VALUE
def sort_event_diffs_by_type(event_diffs):
	for i in range(len(event_diffs)):
		j = i + 1
		while j < len(event_diffs):
			if event_diffs[i].difftype > event_diffs[j].difftype:
				hold = event_diffs[i]
				event_diffs[i] = event_diffs[j]
				event_diffs[j] = hold
			j = j + 1
	return event_diffs

# THIS FUNCTION FORMATS THE CONTENTS OF EVENT_DIFF OBJECTS
# AND APPENDS IT TO THE REPORT LIST
def append_diff_to_report(report, event_diffs, event_type, truncate):
	dt = DiffType()
	report.append('::::::')
	report.append(':::::: {}'.format(event_type))
	report.append(':::::: {:<61} | {:<15} | PATH'.format('DIFFERENCE TYPE', 'FREQ'))
	report.append(':::::: --------------------------------------------------------------|-----------------|------>')

	path_type = 'reg' if event_type.upper().startswith('REGMOD') else 'dir'
	event_diffs = sort_event_diffs_by_type(event_diffs)
	for diff in event_diffs:
		if diff.difftype == dt.MISS_FM_REP:
			report.append(':::::: {:<61} | {:<15} | {}'.format(
				diff.difftype,
				'n/a',
				truncate_path(diff.target_event.path, path_type) if truncate else diff.target_event.path))
		elif diff.difftype == dt.MISS_FM_TAR:
			report.append(':::::: {:<61} | {:<15} | {}'.format(
				diff.difftype,
				'n/a',
				truncate_path(diff.representative_event.path, path_type) if truncate else diff.representative_event.path))
		elif diff.difftype == dt.HIGH_FQ_REP:
			report.append(':::::: {:<61} | {:<15} | {}'.format(
				diff.difftype,
				'{:<6.4f} > {:<6.4f}'.format(float(diff.representative_event.perc), float(diff.target_event.perc)),
				truncate_path(diff.target_event.path, path_type) if truncate else diff.target_event.path))
		elif diff.difftype == dt.HIGH_FQ_TAR:
			report.append(':::::: {:<61} | {:<15} | {}'.format(
				diff.difftype,
				'{:<6.4f} > {:<6.4f}'.format(float(diff.target_event.perc), float(diff.representative_event.perc)),
				truncate_path(diff.target_event.path, path_type) if truncate else diff.target_event.path))
		elif diff.difftype.startswith('TARGET SAMPLE HAS NO'):
			report.append(':::::: {:<61} | {:<15} | {}'.format(
				diff.difftype,
				'n/a',
				truncate_path(diff.representative_event.path, path_type) if truncate else diff.representative_event.path))
		elif diff.difftype.startswith('REPRESENTATIVE SAMPLE HAS NO'):
			report.append(':::::: {:<61} | {:<15} | {}'.format(
				diff.difftype,
				'n/a',
				truncate_path(diff.target_event.path, path_type) if truncate else diff.target_event.path))
	return report

# THIS FUNCTION FORMATS THE CONTENTS OF EVENT_FREQ OBJECTS
# AND APPENDS IT TO THE REPORT LIST
def append_to_report(report, event_freqs, event_type, truncate):
	report.append('::::::')
	report.append(':::::: {}'.format(event_type))
	report.append(':::::: RATIO   | FREQ   | PATH')
	report.append(':::::: --------|--------|------->')
	path_type = 'reg' if event_type.upper().startswith('REGMOD') else 'dir'
	for event in event_freqs:
		report.append(':::::: {:<7} | {:<6.4f} | {}'.format(
			'{}/{}'.format(event.count, event.total),
			event.perc,
			truncate_path(event.path, path_type) if truncate else event.path))
	return report

# THIS IS THE MAIN FUNCTION FOR FORMATTING THE RESULTS
# OF A --BY-PROCESS MODE QUERY INTO A REPORT
def format_report_by_process(params, event_freqs):

	debug(params['verbose'], 'Generating report')
	report = []

	# CHECK THAT THERE ARE, IN FACT, RESULTS TO REPORT
	no_results = True
	for key in event_freqs:
		if not event_freqs[key] == None and len(event_freqs[key]) > 0:
			no_results = False
			break
	# OTHERWISE, YOU SHALL NOT PASS
	if no_results == True:
		report.append('::::::')
		report.append(':::::: NO RESULTS FOR PROCESS {}'.format(params['search_name'].upper()))
		return report

	# APPEND THE REPORT HEADERS TO THE REPORT
	report.append('::::::')
	report.append(':::::: RESULTS FOR PROCESS {}'.format(params['search_name'].upper()))
	report.append('::::::')
	report.append(':::::: FILTERS')
	report.append(':::::: start_time = {}'.format(params['start_time']))
	if params['threshold_lt'] != None:
		report.append(':::::: threshold (less than) = {}%'.format(params['threshold_lt']))
	if params['threshold_gt'] != None:
		report.append(':::::: threshold (greater than) = {}%'.format(params['threshold_gt']))
	report.append(':::::: sample_size = {}'.format(params['sample_size']))
	if not params['user_name'] == None:
		report.append(':::::: user_name = {}'.format(params['user_name']))
	elif not params['exclude_user'] == None:
		report.append(':::::: exclude_user = {}'.format(params['exclude_user']))
	if not params['host_name'] == None:
		report.append(':::::: host_name = {}'.format(params['host_name']))
	elif not params['exclude_host'] == None:
		report.append(':::::: exclude_host = {}'.format(params['exclude_host']))

	# APPEND THE EVENTS TO THE REPORT
	if not event_freqs['modloads'] == None and len(event_freqs['modloads']) > 0:
		report = append_to_report(report, event_freqs['modloads'], 'MODLOADS', params['truncate'])
	if not event_freqs['regmods'] == None and len(event_freqs['regmods']) > 0:
		report = append_to_report(report, event_freqs['regmods'], 'REGMODS', params['truncate'])
	if not event_freqs['childprocs'] == None and len(event_freqs['childprocs']) > 0:
		report = append_to_report(report, event_freqs['childprocs'], 'CHILDPROCS', params['truncate'])
	if not event_freqs['filemods'] == None and len(event_freqs['filemods']) > 0:
		report = append_to_report(report, event_freqs['filemods'], 'FILEMODS', params['truncate'])
	if not event_freqs['netconns'] == None and len(event_freqs['netconns']) > 0:
		report = append_to_report(report, event_freqs['netconns'], 'NETCONNS', params['truncate'])
	if not event_freqs['crossprocs'] == None and len(event_freqs['crossprocs']) > 0:
		report = append_to_report(report, event_freqs['crossprocs'], 'CROSSPROCS', params['truncate'])

	# CLOSE AND RETURN THE REPORT
	report.append('::::::')
	report.append(':::::: END')
	return report

# THIS IS THE MAIN FUNCTION FOR FORMATTING THE RESULTS OF A
# --BY-EVENT QUERY INTO A REPORT
def format_report_by_event(params, event_freqs):

	debug(params['verbose'], 'Generating report')
	report = []

	# GET THE EVENT TYPE THAT WAS IN THE QUERY
	event_type = get_event_type_flags(params)
	if event_type == 'm':
		event_type = 'MODLOAD'
	elif event_type == 'r':
		event_type = 'REGMOD'
	elif event_type == 'f':
		event_type = 'FILEMOD'
	elif event_type == 'c':
		event_type = 'CHILDMOD'
	elif event_type == 'd':
		event_type = 'NETCONN'

	# IF THERE ARE NO RESULTS, GO NO FURTHER
	if len(event_freqs) == 0:
		report.append('::::::')
		report.append(':::::: NO RESULTS FOR {} {}'.format(event_type, params['search_name'].upper()))
		return report

	# APPEND THE REPORT HEADERS TO THE REPORT
	report.append('::::::')
	report.append(':::::: RESULTS FOR {} {}'.format(event_type, params['search_name'].upper()))
	report.append('::::::')
	report.append(':::::: FILTERS')
	report.append(':::::: start_time = {}'.format(params['start_time']))
	if params['threshold_lt'] != None:
		report.append(':::::: threshold (less than) = {}%'.format(params['threshold_lt']))
	if params['threshold_gt'] != None:
		report.append(':::::: threshold (greater than) = {}%'.format(params['threshold_gt']))
	report.append(':::::: sample_size = {}'.format(params['sample_size']))
	if not params['user_name'] == None:
		report.append(':::::: user_name = {}'.format(params['user_name']))
	elif not params['exclude_user'] == None:
		report.append(':::::: exclude_user = {}'.format(params['exclude_user']))
	if not params['host_name'] == None:
		report.append(':::::: host_name = {}'.format(params['host_name']))
	elif not params['exclude_host'] == None:
		report.append(':::::: exclude_host = {}'.format(params['exclude_host']))

	# APPEND THE PROCESSES TO THE REPORT
	report = append_to_report(report, event_freqs, 'PROCESSES', params['truncate'])

	# CLOSE AND RETURN THE REPORT
	report.append('::::::')
	report.append(':::::: END')
	return report

# THIS FUNCTION TAKES A FILE NAME AND USES IT TO
# CREATE A PARAMS JSON CONTAINER TO HOLD THE PARAMETERS
# USED IN THE SEARCH THAT GENERATED THE FILE
def get_params_from_file_name(file_name):
	# INITIALIZE JSON CONTAINER FOR PARAMETERS
	params = {
		'search_name':None,
		'regmods':False,
		'filemods':False,
		'modloads':False,
		'crossprocs':False,
		'childprocs':False,
		'netconns':False,
		'sample_size':10,
		'user_name':None,
		'exclude_user':None,
		'host_name':None,
		'exclude_host':None,
		'threshold_lt':100,
		'threshold_gt':0,
		'event_type':None
		}

	# SPLIT THE NAME AT THE DEFINED FLAGS, INTO AN ARRAY
	name_ary = re.split(r'_[hHuUtTnse]{1}-', file_name[0:len(file_name) - 4]) # trim .csv
	params['search_name'] = name_ary[1].replace('__BS__', '\\')
	params['sample_size'] = int(name_ary[3])
	params['threshold_lt'] = int(name_ary[4]) if not name_ary[4] == "None" else "None"
	params['threshold_gt'] = int(name_ary[5]) if not name_ary[5] == "None" else "None"
	flags = name_ary[2]
	if 'r' in flags:
		params['regmods'] = True
		params['event_type'] = 'REGMOD'
	if 'f' in flags:
		params['filemods'] = True
		params['event_type'] = 'FILEMOD'
	if 'm' in flags:
		params['modloads'] = True
		params['event_type'] = 'MODLOAD'
	if 'x' in flags:
		params['crossprocs'] = True
		params['event_type'] = 'CROSSPROC'
	if 'c' in flags:
		params['childprocs'] = True
		params['event_type'] = 'CHILDPROC'
	if 'd' in flags:
		params['netconns'] = True
		params['event_type'] = 'NETCONN'
	# IF THERE ARE 8 ELEMENTS IN THE NAME_ARY, THEN A USER FLAG AND A HOST FLAG WAS USED
	if len(name_ary) == 8:
		if '_u-' in file_name:
			params['user_name'] = name_ary[6]
		elif '_U-' in file_name:
			params['exclude_user'] = name_ary[6]
		if '_h-' in file_name:
			params['host_name'] = name_ary[7]
		elif '_H-' in file_name:
			params['exclude_host'] = name_ary[7]
	# IF THERE ARE 7 ELEMENTS IN THE NAME_ARY, THEN EITHER A USER FLAG OR A HOST FLAG WAS USED
	elif len(name_ary) == 7:
		if '_h-' in file_name:
			params['host_name'] = name_ary[6]
		elif '_H-' in file_name:
			params['exclude_host'] = name_ary[6]
		elif '_u-' in file_name:
			params['user_name'] = name_ary[6]
		elif '_U-' in file_name:
			params['exclude_user'] = name_ary[6]
	return params

# THIS IS THE MAIN FUNCTION FOR FORMATTING THE RESULTS OF A
# --COMPARE-PROCESS MODE QUERY INTO A REPORT
def format_report_compare_process(params, event_freqs):
	report = []

	# CHECK THAT THERE ARE INDEED RESULTS
	no_results = True
	for key in event_freqs:
		if not event_freqs[key] == None and len(event_freqs[key]) > 0:
			no_results = False
			break
	# IF NOT, GO NO FURTHER
	if no_results:
		report.append('::::::')
		report.append(':::::: TARGET AND REPRESENTATIVE SAMPLES ARE THE SAME')
		return report

	# EXTRACT THE PARAMETERS USED TO GENERATE THE REPRESENTATIVE SAMPLE FROM ITS FILE NAME
	representative_sample_params = get_params_from_file_name(params['import_sample'])

	# APPEND THE REPORT HEADERS TO THE REPORT
	report.append('::::::')
	report.append(':::::: PROCESS COMPARISON RESULTS')
	report.append('::::::')
	report.append(':::::: REPRESENTATIVE SAMPLE PROCESS {}'.format(representative_sample_params['search_name'].upper()))
	report.append(':::::: file = {}'.format(params['import_sample'].replace('__BS__', '\\')))
	if representative_sample_params['threshold_lt'] != None:
		report.append(':::::: threshold (less than) = {}%'.format(representative_sample_params['threshold_lt']))
	if representative_sample_params['threshold_gt'] != None:
		report.append(':::::: threshold (greather than) = {}%'.format(representative_sample_params['threshold_gt']))
	report.append(':::::: sample_size = {}'.format(representative_sample_params['sample_size']))
	if not representative_sample_params['user_name'] == None:
		report.append(':::::: user_name = {}'.format(representative_sample_params['user_name']))
	elif not representative_sample_params['exclude_user'] == None:
		report.append(':::::: exclude_user = {}'.format(representative_sample_params['exclude_user']))
	if not representative_sample_params['host_name'] == None:
		report.append(':::::: host_name = {}'.format(representative_sample_params['host_name']))
	elif not representative_sample_params['exclude_host'] == None:
		report.append(':::::: exclude_host = {}'.format(representative_sample_params['exclude_host']))
	report.append('::::::')
	report.append(':::::: TARGET SAMPLE PROCESS {}'.format(params['search_name'].upper()))
	report.append(':::::: start_time = {}'.format(params['start_time']))
	if params['threshold_lt'] != None:
		report.append(':::::: threshold (less than) = {}%'.format(params['threshold_lt']))
	if params['threshold_gt'] != None:
		report.append(':::::: threshold (greater than) = {}%'.format(params['threshold_gt']))
	report.append(':::::: sample_size = {}'.format(params['sample_size']))
	if not params['user_name'] == None:
		report.append(':::::: user_name = {}'.format(params['user_name']))
	elif not params['exclude_user'] == None:
		report.append(':::::: exclude_user = {}'.format(params['exclude_user']))
	if not params['host_name'] == None:
		report.append(':::::: host_name = {}'.format(params['host_name']))
	elif not params['exclude_host'] == None:
		report.append(':::::: exclude_host = {}'.format(params['exclude_host']))

	# APPEND THE EVENT DIFFERENCES TO THE REPORT
	for key in event_freqs:
		if not event_freqs[key] == None and len(event_freqs[key]) > 0:
			report = append_diff_to_report(report, event_freqs[key], key.upper(), params['truncate'])

	# CLOSE AND RETURN REPORT
	report.append('::::::')
	report.append(':::::: END')
	return report

# THIS IS THE MAIN FUNCTION FOR FORMATTING THE RESULTS
# OF A --COMPARE-EVENT QUERY INTO A REPORT
def format_report_compare_event(params, event_freqs):
	report = []

	# IF THERE ARE NO RESULTS, GO NO FURTHER
	if len(event_freqs) == 0:
		report.append('::::::')
		report.append(':::::: TARGET AND REPRESENTATIVE SAMPLES ARE THE SAME')
		return report

	# GET THE EVENT TYPE USED IN THE TARGET SAMPLE
	event_type = 'NETCONN'
	if params['modloads'] == True:
		event_type = 'MODLOAD'
	elif params['regmods'] == True:
		event_type = 'REGMOD'
	elif params['filemods'] == True:
		event_type = 'FILEMOD'
	elif params['childprocs'] == True:
		event_type = 'CHILDPROC'

	# EXTRACT THE PARAMETERS USED TO GENERATE THE REPRESENTATIVE SAMPLE FROM THE FILE NAME
	representative_sample_params = get_params_from_file_name(params['import_sample'])

	# APPEND THE REPORT HEADERS TO THE REPORT
	report.append('::::::')
	report.append(':::::: EVENT COMPARISON RESULTS')
	report.append('::::::')
	report.append(':::::: REPRESENTATIVE SAMPLE {} {}'.format(representative_sample_params['event_type'], representative_sample_params['search_name'].upper()))
	report.append(':::::: file = {}'.format(params['import_sample'].replace('__BS__', '\\')))
	report.append(':::::: threshold (less than) = {}'.format(representative_sample_params['threshold_lt']))
	report.append(':::::: threshold (greater than) = {}'.format(representative_sample_params['threshold_gt']))
	report.append(':::::: sample_size = {}'.format(representative_sample_params['sample_size']))
	if not representative_sample_params['user_name'] == None:
		report.append(':::::: user_name = {}'.format(representative_sample_params['user_name']))
	elif not representative_sample_params['exclude_user'] == None:
		report.append(':::::: exclude_user = {}'.format(representative_sample_params['exclude_user']))
	if not representative_sample_params['host_name'] == None:
		report.append(':::::: host_name = {}'.format(representative_sample_params['host_name']))
	elif not representative_sample_params['exclude_host'] == None:
		report.append(':::::: exclude_host = {}'.format(representative_sample_params['exclude_host']))
	report.append('::::::')
	report.append(':::::: TARGET SAMPLE {} {}'.format(event_type, params['search_name'].upper()))
	report.append(':::::: start_time = {}'.format(params['start_time']))
	report.append(':::::: threshold (less than) = {}'.format(params['threshold_lt']))
	report.append(':::::: threshold (greater than) = {}'.format(params['threshold_gt']))
	report.append(':::::: sample_size = {}'.format(params['sample_size']))
	if not params['user_name'] == None:
		report.append(':::::: user_name = {}'.format(params['user_name']))
	elif not params['exclude_user'] == None:
		report.append(':::::: exclude_user = {}'.format(params['exclude_user']))
	if not params['host_name'] == None:
		report.append(':::::: host_name = {}'.format(params['host_name']))
	elif not params['exclude_host'] == None:
		report.append(':::::: exclude_host = {}'.format(params['exclude_host']))

	# APPEND THE PROCESS DIFFERENCES TO THE REPORT
	report = append_diff_to_report(report, event_freqs, 'PROCESSES', params['truncate'])

	# CLOSE AND RETURN THE REPORT
	report.append('::::::')
	report.append(':::::: END')
	return report

# THIS FUNCTION EXTRACTS THE EVENT TYPE FLAGS
# FROM PARAMS AND RETURN THEM IN A STRING
def get_event_type_flags(params):
	flags = ""
	if params['modloads'] == True:
		flags += 'm'
	if params['regmods'] == True:
		flags += 'r'
	if params['filemods'] == True:
		flags += 'f'
	if params['crossprocs'] == True:
		flags += 'x'
	if params['childprocs'] == True:
		flags += 'c'
	if params['netconns'] == True:
		flags += 'd'
	return flags

# THIS FUNCTION IMPORTS THE CONTENTS OF A SAMPLE FILE
# AND RETURNS THEM AS A LIST OR DICTIONARY OF EVENT_FREQ OBJECTS
def import_sample(params):
	# INITIALIZE THE COLUMN INDEX VARIABLES
	search_name = None
	event_type = None
	path = None
	count = None
	total = None
	freq = None

	# CHECK THAT THE USER HASN'T ACCIDENTALLY COPIED DIRECTORY SECTIONS INTO THE FILEPATH
	file_path = ''
	if '/' in params['import_sample']:
		hold = params['import_sample'].split('/')
		params['import_sample'] = hold[len(hold) - 1]

	# FORMAT THE IMPORT_SAMPLE VALUE AS A FILE PATH
	file_path = './samples/{}/{}'.format('process' if params['mode'] == 'COMPARE_PROCESS' else 'event', params['import_sample'])

	# THE COLUMN IN THE .CSV FILE DIFFERS DEPENDING UPON WHETHER THE
	# FILE IS THE RESULT OF A PROCESS OR EVENT SEARCH
	if params['mode'] == 'COMPARE_PROCESS':
		search_name = 0
		event_type = 1
		path = 2
		count = 3
		total = 4
		freq = 5
	elif params['mode'] == 'COMPARE_EVENT':
		search_name = 0
		path = 1
		count = 2
		total = 3
		freq = 4

	# INITIALIZE THE EVENT_FREQS CONTAINER AS A LIST (FOR --COMPARE-EVENT) OR DICTIONARY (FOR --COMPARE-PROCESS)
	event_freqs = [] if event_type == None else {'modloads':None, 'regmods':None, 'childprocs':None, 'filemods':None, 'netconns':None, 'crossprocs':None}
	file = open(file_path, 'r')
	skip = True
	for line in file:
		# SKIP THE FIRST LINE IN THE FILE (I.E. THE HEADERS)
		if skip:
			skip = False
		else:
			# EXTRACT THE VALUES FROM THE LINE IN THE FILE AND REMOVE QUOTATION MARKS
			path_value = line.split(',')[path].replace('\'','')
			freq_value = line.split(',')[freq].replace('\'','')
			freq_value = freq_value.replace('\n','')
			count_value = line.split(',')[count].replace('\'','')
			total_value = line.split(',')[total].replace('\'','')
			if not event_type == None:
				# IF RUNNING IN --COMPARE-PROCESS MODE, RETRIEVE THE EVENT_TYPE VALUE
				event_type_value = '{}s'.format(line.split(',')[event_type].replace('\'',''))
				if not event_freqs[event_type_value] == None:
					event_freqs[event_type_value].append(EventFreq(path_value, freq_value, count_value, total_value))
				else:
					event_freqs[event_type_value] = []
					event_freqs[event_type_value].append(EventFreq(path_value, freq_value, count_value, total_value))
			else:
				event_freqs.append(EventFreq(path_value, freq_value, count_value, total_value))
	file.close()
	return event_freqs

# THIS FUNCTION CHECKS WHETHER THE NECESSARY FILE DIRECTORIES
# EXISTS IN THE JETFREQ WORKING DIRECTORY AND CREATES THEM
# IF NECESSARY
def check_for_sample_dir():
	dirs = [
		"./samples",
		"./samples/process",
		"./samples/event",
		"./samples/process/diff",
		"./samples/event/diff"
	]
	for d in dirs:
		try:
			os.mkdir(d)
		except OSError:
			pass

# THIS FUNCTION FORMATS THE CURRENT DATE TIME INTO
# A STRING APPROPRIATE FOR A FILE NAME
def format_datetime():
	dt = str(datetime.now())

	# REMOVE COLONS AND PERIODS
	dt = re.sub(r'(:|\.)', '-', dt)

	# REMOVE WHITESPACE
	dt = re.sub(r'\s', '_', dt)

	return dt

# THIS FUNCTION CHECKS THE JETFREQ MODE AND CREATES
# AN APPROPRIATE SUB DIRECTORY PREFIX FOR THE FILENAME
def build_sub_dir(params):
	if params['mode'] == 'BY_EVENT':
		return 'event'
	elif params['mode'] == 'COMPARE_PROCESS':
		return 'process/diff'
	elif params['mode'] == 'COMPARE_EVENT':
		return 'event/diff'
	return 'process'

# THIS FUNCTION AUTOMATICALLY GENERATES A FILEPATH IN
# WHICH TO SAVE THE CONTENTS OF A JETFREQ QUERY
def build_file_path(params):

	debug(params['verbose'], 'Creating the file path')

	# GENERATE THE BASE NAME OF THE FILE
	file_path = './samples/{}/{}_s-{}_e-{}_n-{}'.format(
		build_sub_dir(params),
		format_datetime(),
		params['search_name'].replace('\\', '__BS__'),
		get_event_type_flags(params),
		params['sample_size']
	)
	if params['threshold_lt'] == None:
		file_path += "_t-None"
	else:
		file_path += "_t-{}".format(params['threshold_lt'])
	if params['threshold_gt'] == None:
		file_path += "_T-None"
	else:
		file_path += "_T-{}".format(params['threshold_gt'])

	# APPEND THE USER NAME FILTERS, IF THEY EXIST
	if params['user_name'] != None:
		file_path += '_u-{}'.format(params['user_name'].lower())
	elif params['exclude_user'] != None:
		file_path += '_U-{}'.format(params['exclude_user'].lower())

	# APPEND THE HOST NAME FILTERS, IF THEY EXIST
	if params['host_name'] != None:
		file_path += '_h-{}'.format(params['host_name'].lower())
	elif params['exclude_host'] != None:
		file_path += '_H-{}'.format(params['exclude_host'].lower())

	# APPEND THE FILE NAME FOR THE REPRESENTATIVE SAMPLE IF WRITING A 'COMPARE' MODE REPORT
	if not re.match(r'^COMPARE_', params['mode']) == None:
		dt = params['import_sample']
		if '/' in params['import_sample']:
			fn_ary = params['import_sample'].split('/')
			dt = fn_ary[len(fn_ary) - 1]
		dt = re.split(r'_s-', dt)[0]
		file_path += '_i-{}'.format(dt)

	# CLOSE FILE PATH WITH CSV EXTENSION
	file_path += '.csv'

	# ALTHOUGH HIGHLY UNLIKELY BECAUSE OF THE TIME STAMP, IF THE FILE
	# ALREADY EXISTS, APPEND A NUMERAL ON THE END OF THE FILENAME
	sn = 0
	while os.path.isfile(file_path):
		sn = sn + 1
		file_path = file_path[0:file_path.index('.csv')]
		file_path += '_' + sn + '.csv'

	debug(params['verbose'], 'File path is {}'.format(file_path))

	return file_path

# THIS IS THE MAIN FUNCTION FOR WRITING THE RESULTS OF A --COMPARE-PROCESS
# QUERY TO FILE
def out_file_compare_process(params, event_diffs):
	# CHECK THAT THERE ARE INDEED RESULTS
	no_results = True
	for key in event_diffs:
		if len(event_diffs[key]) > 0:
			no_results = False
	# IF NOT, GO NO FURTHER
	if no_results:
		raise jfexceptions.NoDiffsFoundError(params['search_name'], params['import_sample'])

	# CHECK THAT THE APPROPRIATE DIRECTORIES EXIST
	# GENERATE THE FILE PATH AND RETRIEVE THE REPRESENTATIVE SAMPLE PARAMETERS
	check_for_sample_dir()
	file_path = build_file_path(params)
	rep_params = get_params_from_file_name(params['import_sample'])
	difftype = DiffType()

	debug(params['verbose'], 'Writing to file {}'.format(file_path))

	# WRITE THE EVENT DIFFERENCES TO FILE
	file = open(file_path, 'w')
	file.write('\'event_type\',\'diff_type\',\'tar_process\',\'tar_event\',\'tar_freq\',\'tar_count\',\'tar_total\',\'rep_process\',\'rep_event\',\'rep_freq\',\'rep_count\',\'rep_total\'\n')
	for key in event_diffs:
		if len(event_diffs[key]) > 0:
			event_diffs_for_type = sort_event_diffs_by_type(event_diffs[key])
			for event_diff in event_diffs_for_type:
				if event_diff.difftype == difftype.MISS_FM_REP or not re.match(r'^REPRESENTATIVE SAMPLE HAS NO', event_diff.difftype) == None:
					file.write('\'{}\',\'{}\',\'{}\',\'{}\',\'{}\',\'{}\',\'{}\',\'{}\',\'NA\',\'NA\',\'NA\',\'NA\'\n'.format(
						key,
						event_diff.difftype,
						params['search_name'],
						event_diff.target_event.path,
						event_diff.target_event.perc,
						event_diff.target_event.count,
						event_diff.target_event.total,
						rep_params['search_name']
						)
					)
				elif event_diff.difftype == difftype.MISS_FM_TAR or not re.match(r'^TARGET SAMPLE HAS NO', event_diff.difftype) == None:
					file.write('\'{}\',\'{}\',\'{}\',\'NA\',\'NA\',\'NA\',\'NA\',\'{}\',\'{}\',\'{}\',\'{}\',\'{}\'\n'.format(
						key,
						event_diff.difftype,
						params['search_name'],
						rep_params['search_name'],
						event_diff.representative_event.path,
						event_diff.representative_event.perc,
						event_diff.representative_event.count,
						event_diff.representative_event.total
						)
					)
				else:
					file.write('\'{}\',\'{}\',\'{}\',\'{}\',\'{}\',\'{}\',\'{}\',\'{}\',\'{}\',\'{}\',\'{}\',\'{}\'\n'.format(
						key,
						event_diff.difftype,
						params['search_name'],
						event_diff.target_event.path,
						event_diff.target_event.perc,
						event_diff.target_event.count,
						event_diff.target_event.total,
						rep_params['search_name'],
						event_diff.representative_event.path,
						event_diff.representative_event.perc,
						event_diff.representative_event.count,
						event_diff.representative_event.total
						)
					)
	file.close()
	return file_path

# THIS IS THE MAIN FUNCTION FOR WRITING THE RESULTS OF
# A --COMPARE-EVENT QUERY TO FILE
def out_file_compare_event(params, event_diffs):
	# IF THERE ARE NO RESULTS, GO NO FURTHER
	if len(event_diffs) == 0:
		raise NoDiffsFoundError(params['search_name'], params['import_sample'])

	# CHECK THAT THE APPROPRIATE DIRECTORY EXISTS, GENERATE
	# THE FILE NAME AND RETRIEVE THE PARAMETERS USED IN THE
	# REPRESENTATIVE SAMPLE
	check_for_sample_dir()
	file_path = build_file_path(params)
	rep_params = get_params_from_file_name(params['import_sample'])
	difftype = DiffType()
	event_diffs = sort_event_diffs_by_type(event_diffs)

	# RETRIEVE THE EVENT TYPE USED IN THE QUERY
	target_event_type = 'modload'
	if params['regmods'] == True:
		target_event_type = 'regmod'
	elif params['filemods'] == True:
		target_event_type = 'filemod'
	elif params['childprocs'] == True:
		target_event_type = 'childproc'
	elif params['netconns'] == True:
		target_event_type = 'netconn'

	debug(params['verbose'], 'Writing to file {}'.format(file_path))

	# WRITE TO FILE
	file = open(file_path, 'w')
	file.write('\'diff_type\',\'tar_event\',\'tar_process\',\'tar_freq\',\'tar_count\',\'tar_total\',\'rep_event\',\'rep_process\',\'rep_freq\',\'rep_count\',\'rep_total\'\n'.format())
	for event_diff in event_diffs:
		if event_diff.difftype == difftype.MISS_FM_REP:
			file.write('\'{}\',\'{}\',\'{}\',\'{}\',\'{}\',\'{}\',\'{}\',\'NA\',\'NA\',\'NA\',\'NA\'\n'.format(
				event_diff.difftype,
				target_event_type,
				event_diff.target_event.path,
				event_diff.target_event.perc,
				event_diff.target_event.count,
				event_diff.target_event.total,
				rep_params['event_type'].lower()
				)
			)
		elif event_diff.difftype == difftype.MISS_FM_TAR:
			file.write('\'{}\',\'{}\',\'NA\',\'NA\',\'NA\',\'NA\',\'{}\',\'{}\',\'{}\',\'{}\',\'{}\'\n'.format(
				event_diff.difftype,
				target_event_type,
				rep_params['event_type'].lower(),
				event_diff.representative_event.path,
				event_diff.representative_event.perc,
				event_diff.representative_event.count,
				event_diff.representative_event.total
				)
			)
		else: #difftype.HIGH_FQ_*
			file.write('\'{}\',\'{}\',\'{}\',\'{}\',\'{}\',\'{}\',\'{}\',\'{}\',\'{}\',\'{}\',\'{}\'\n'.format(
				event_diff.difftype,
				target_event_type,
				event_diff.target_event.path,
				event_diff.target_event.perc,
				event_diff.target_event.count,
				event_diff.target_event.total,
				rep_params['event_type'].lower(),
				event_diff.representative_event.path,
				event_diff.representative_event.perc,
				event_diff.representative_event.count,
				event_diff.representative_event.total
				)
			)
	file.close()
	return file_path

# THIS IS THE MAIN FUNCTION FOR WRITING THE RESULTS OF A
# --BY-EVENT QUERY TO FILE
def out_file_by_event(params, event_freqs):
	# IF THERE ARE NO RESULTS, GO NO FURTHER
	if len(event_freqs) == 0:
		raise jfexceptions.NoEventsFoundError(params['search_name'])

	# CHECK THAT THE APPROPRIATE DIRECTORY EXISTS AND
	# GENERATE THE FILE PATH
	check_for_sample_dir()
	file_path = build_file_path(params)

	# GET THE EVENT TYPE USED IN THE QUERY
	event_type = 'modload'
	if params['filemods'] == True:
		event_type = 'filemod'
	elif params['regmods'] == True:
		event_type = 'regmod'
	elif params['childprocs'] == True:
		event_type = 'childproc'
	elif params['netconns'] == True:
		event_type = 'netconn'

	debug(params['verbose'], 'Writing to file {}'.format(file_path))

	# WRITE TO FILE
	file = open(file_path, 'w')
	file.write('\'{}\',\'path\',\'count\',\'total\',\'frequency\'\n'.format(event_type))
	for event in event_freqs:
		file.write('\'{}\',\'{}\',\'{}\',\'{}\',\'{}\'\n'.format(params['search_name'], event.path, event.count, event.total, event.perc))
	file.close()
	return file_path

# THIS IS THE MAIN FUNCTION FOR WRITING THE RESULTS
# OF A --BY-PROCESS QUERY TO FILE
def out_file_by_process(params, event_freqs):
	# CHECK THAT THERE ARE INDEED RESULTS
	no_results = True
	for key in event_freqs:
		if not event_freqs[key] == None:
			no_results = False
	# IF NOT, THOU SHALT NOT PASSETH
	if no_results:
		raise jfexceptions.NoEventsFoundError(params['search_name'])

	# CHECK THAT THE APPROPRIATE DIRECTORY EXISTS
	# AND GENERATE THE FILE PATH
	check_for_sample_dir()
	file_path = build_file_path(params)

	debug(params['verbose'], 'Writing to file {}'.format(file_path))

	# WRITE TO FILE
	file = open(file_path, 'w')
	file.write('\'process\',\'event_type\',\'path\',\'count\',\'total\',\'freq\'\n')
	if not event_freqs['modloads'] == None:
		events = sort_events(event_freqs['modloads'])
		for event in events:
			file.write('\'{}\',\'modload\',\'{}\',\'{}\',\'{}\',\'{}\'\n'.format(params['search_name'], event.path, event.count, event.total, event.perc))
	if not event_freqs['regmods'] == None:
		events = sort_events(event_freqs['regmods'])
		for event in events:
			file.write('\'{}\',\'regmod\',\'{}\',\'{}\',\'{}\',\'{}\'\n'.format(params['search_name'], event.path, event.count, event.total, event.perc))
	if not event_freqs['childprocs'] == None:
		events = sort_events(event_freqs['childprocs'])
		for event in events:
			file.write('\'{}\',\'childproc\',\'{}\',\'{}\',\'{}\',\'{}\'\n'.format(params['search_name'], event.path, event.count, event.total, event.perc))
	if not event_freqs['filemods'] == None:
		events = sort_events(event_freqs['filemods'])
		for event in events:
			file.write('\'{}\',\'filemod\',\'{}\',\'{}\',\'{}\',\'{}\'\n'.format(params['search_name'], event.path, event.count, event.total, event.perc))
	if not event_freqs['netconns'] == None:
		events = sort_events(event_freqs['netconns'])
		for event in events:
			file.write('\'{}\',\'netconn\',\'{}\',\'{}\',\'{}\',\'{}\'\n'.format(params['search_name'], event.path, event.count, event.total, event.perc))
	if not event_freqs['crossprocs'] == None:
		events = sort_events(event_freqs['crossprocs'])
		for event in events:
			file.write('\'{}\',\'crossproc\',\'{}\',\'{}\',\'{}\',\'{}\'\n'.format(params['search_name'], event.path, event.count, event.total, event.perc))
	file.close()
	return file_path

# THIS FUNCTION READS THE README.TXT FILE
# OUT OUTPUTS THE USAGE SECTION TO STDOUT
def show_usage():
	h = open("README.txt", "r")
	usage = ""
	capture = False
	for line in h:
		if capture == False and len(usage) > 0:
			break
		elif capture == True:
			usage += line
			if re.match(r'^Show Help :', line) != None:
				capture = False
		elif re.match(r'^Usage:', line) != None:
			capture = True
	print("\nUSAGE:\n{}".format(usage))

# THIS FUNCTION WRITES THE CONTENTS OF THE README.TXT
# FILE TO STDOUT
def show_help():
	h = open("README.txt", "r")
	print(h.read())

# THIS FUNCTION IMPORTS THE CONTENTS OF THE CONF FILE
# AND ADDS THEM TO THE PARAMS DICTIONARY
def import_conf(params):
	debug(params['verbose'], 'Importing configuration file')
	conf = open('./conf/jetfreq.cfg', 'r')
	for line in conf:
		if line.startswith('#'):
			pass
		else:
			params[line.split('=')[0].lower().strip()] = line.split('=')[1].strip()
	conf.close()
	return params

# THIS FUNCTION CHECKS FOR USER SPECIFIC REG KEY ADDRESSES
# OR USER SPECIFIC FILE DIRECTORIES AND REPLACES THE UNIQUE
# STRINGS WITH A GENERIC STRING, E.G. 'BROOKES' TO '<USER>'.
# THIS ENSURES THAT THE SAME FILE OR REG KEY IN DIFFERENT USERS
# PROFILES OR MACHINES ARE CONSIDERED THE SAME FILE OR REG KEY
# BY JETFREQ.
def homogenize_path(path, path_type, homogenize):
	if not homogenize:
		return path

	if path_type == "reg":
		# \registry\...\usersettings\<sid>\\
		# \registry\user\<sid>\
		if path.lower().startswith('\\registry\\') and '\\usersettings\\' in path:
			try:
				path_ary = path.split('\\')
				sid = False
				for i in range(len(path_ary)):
					if sid == True:
						path_ary[i] = "<SID>"
						sid = False
						break
					elif path_ary[i] == "usersettings":
						sid = True
				return '\\'.join(path_ary)
			except:
				debug(True, 'Error homogenizing regmod path {}'.format(path))
				return path
		elif path.lower().startswith('\\registry\\user\\'):
			try:
				path_ary = path.split('\\')
				path_ary[3] = '<SID>'
				return '\\'.join(path_ary)
			except:
				debug(True, 'Error homogenizing regmod path {}'.format(path))
		else:
			return path
	elif path_type == "dir":
		# c:\users\<user>\
		if path.lower().startswith('c:\\users\\') or path.lower().startswith('\\users\\'):
			try:
				path_ary = path.split('\\')
				path_ary[2] = '<USER>'
				return '\\'.join(path_ary)
			except:
				debug(True, 'Error homogenizing path {}'.format(path))
				return path
		else:
			return path
	else:
		return path

# THIS FUNCTION EXTRACTS THE PATH OF AN EVENT FROM THE
# DATA RETURNED BY THE CARBON BLACK REST API. AS EACH
# EVENT TYPE HAS A UNIQUE FORMAT, THE COLUMN MUST
# BE DEFINED.
def get_event_paths(events, col, path_type, homogenize):
	paths = []
	for event in events:
		paths.append(homogenize_path(event.split('|')[col], path_type, homogenize))
	return paths

# THIS FUNCTION CHECKS FOR ATTEMPTS TO GENERATE VERY LARGE
# SAMPLE SIZES AND WARNS THE USER THAT DUE THE VOLUME OF
# DATA, A QUERY WITH A LARGE SAMPLE SIZE MAY TAKE SOME TIME
# TO GENERATE
def throttle(params):
	if params['mode'].upper().endswith('HELP'):
		return
	elif (params['mode'].upper().endswith('PROCESS') and int(params['sample_size']) > 50) or (params['mode'].upper().endswith('EVENT') and int(params['sample_size']) >= 1000):
		debug(True, 'Consider reducing the sample size.')
		debug(True, 'A sample size of {} in {} mode may generate a large amount of data and require a significant amount of time to process.'.format(params['sample_size'], params['mode']))
		ans = raw_input('jetfreq.py: Would you like to continue (y/n)? ')
		if ans.lower() == 'y':
			return
		elif ans.lower() == 'n':
			exit()
		else:
			debug(True, 'Invalid response. Aborting.')
			exit()


import argparse
from model import *
from computeloss import build_loss
from heads import *
import torch
import copy
from dataloader import *
from utils import *

class LearningRateLambda():
    def __init__(self, decay_schedule, *,
                 decay_factor=0.1,
                 decay_epochs=1.0,
                 warm_up_start_epoch=0,
                 warm_up_epochs=2.0,
                 warm_up_factor=0.01,
                 warm_restart_schedule=None,
                 warm_restart_duration=0.5):
        self.decay_schedule = decay_schedule
        self.decay_factor = decay_factor
        self.decay_epochs = decay_epochs
        self.warm_up_start_epoch = warm_up_start_epoch
        self.warm_up_epochs = warm_up_epochs
        self.warm_up_factor = warm_up_factor
        self.warm_restart_schedule = warm_restart_schedule
        self.warm_restart_duration = warm_restart_duration

    def __call__(self, step_i):
        lambda_ = 1.0

        if step_i <= self.warm_up_start_epoch:
            lambda_ *= self.warm_up_factor
        elif self.warm_up_start_epoch < step_i < self.warm_up_start_epoch + self.warm_up_epochs:
            lambda_ *= self.warm_up_factor**(
                1.0 - (step_i - self.warm_up_start_epoch) / self.warm_up_epochs
            )

        for d in self.decay_schedule:
            if step_i >= d + self.decay_epochs:
                lambda_ *= self.decay_factor
            elif step_i > d:
                lambda_ *= self.decay_factor**(
                    (step_i - d) / self.decay_epochs
                )

        for r in self.warm_restart_schedule:
            if r <= step_i < r + self.warm_restart_duration:
                lambda_ = lambda_**(
                    (step_i - r) / self.warm_restart_duration
                )

        return lambda_

if __name__=='__main__':
	parser = argparse.ArgumentParser()
	parser.add_argument('-o', '--output', default=None,
	                    help='output file')
	parser.add_argument('--disable-cuda', action='store_true',
	                    help='disable CUDA')
	parser.add_argument('--ddp', default=False, action='store_true',
	                    help='[experimental] DistributedDataParallel')
	parser.add_argument('--local_rank', default=None, type=int,
	                    help='[experimental] for torch.distributed.launch')
	parser.add_argument('--no-sync-batchnorm', dest='sync_batchnorm',
	                    default=True, action='store_false',
	                    help='[experimental] in ddp, to not use syncbatchnorm')
	parser.add_argument('--momentum', type=float, default=0.9,
	                   help='SGD momentum, beta1 in Adam')
	parser.add_argument('--beta2', type=float, default=0.999,
	                   help='beta2 for Adam/AMSGrad')
	parser.add_argument('--adam-eps', type=float, default=1e-6,
	                   help='eps value for Adam/AMSGrad')
	parser.add_argument('--no-nesterov', dest='nesterov', default=True, action='store_false',
	                   help='do not use Nesterov momentum for SGD update')
	parser.add_argument('--weight-decay', type=float, default=0.0,
	                   help='SGD/Adam/AMSGrad weight decay')
	parser.add_argument('--adam', action='store_true',
	                   help='use Adam optimizer')
	parser.add_argument('--amsgrad', action='store_true',
	                   help='use Adam optimizer with AMSGrad option')

	parser.add_argument_group('learning rate scheduler')
	parser.add_argument('--lr', type=float, default=1e-3,
	                     help='learning rate')
	parser.add_argument('--lr-decay', default=[], nargs='+', type=float,
	                     help='epochs at which to decay the learning rate')
	parser.add_argument('--lr-decay-factor', default=0.1, type=float,
	                     help='learning rate decay factor')
	parser.add_argument('--lr-decay-epochs', default=1.0, type=float,
	                     help='learning rate decay duration in epochs')
	parser.add_argument('--lr-warm-up-start-epoch', default=0, type=float,
	                     help='starting epoch for warm-up')
	parser.add_argument('--lr-warm-up-epochs', default=1, type=float,
	                     help='number of epochs at the beginning with lower learning rate')
	parser.add_argument('--lr-warm-up-factor', default=0.001, type=float,
	                     help='learning pre-factor during warm-up')
	parser.add_argument('--lr-warm-restarts', default=[], nargs='+', type=float,
	                     help='list of epochs to do a warm restart')
	parser.add_argument('--lr-warm-restart-duration', default=0.5, type=float,
	                     help='duration of a warm restart')

	parser.add_argument('--batch_size', default=1, type=int)
	parser.add_argument('--loader_workers', default=4, type=int)

	Trainer.cli(parser)
	args = parser.parse_args()
	Trainer.configure(args)

	train_loader = Cocodata(args).train_loader()
	val_loader = Cocodata(args).train_loader()
	print(len(train_loader))
	print(len(val_loader))
	model = build_model().cuda()

	loss = build_loss().loss(get_coco_multihead())
	optimizer = torch.optim.Adam(
			model.parameters(),
            lr=args.lr, betas=(args.momentum, args.beta2),
            weight_decay=args.weight_decay, eps=args.adam_eps, amsgrad=args.amsgrad)
	training_batches_per_epoch = len(train_loader)
	start_epoch = 0
	lr_scheduler = torch.optim.lr_scheduler.LambdaLR(
		optimizer,
		[
				LearningRateLambda(
				[s * training_batches_per_epoch for s in args.lr_decay],
				decay_factor=args.lr_decay_factor,
				decay_epochs=args.lr_decay_epochs * training_batches_per_epoch,
				warm_up_start_epoch=args.lr_warm_up_start_epoch * training_batches_per_epoch,
				warm_up_epochs=args.lr_warm_up_epochs * training_batches_per_epoch,
				warm_up_factor=args.lr_warm_up_factor,
				warm_restart_schedule=[r * training_batches_per_epoch
				for r in args.lr_warm_restarts],
				warm_restart_duration=args.lr_warm_restart_duration * training_batches_per_epoch,
				),
		],
		last_epoch= start_epoch  * training_batches_per_epoch - 1,
		)
	checkpoint_shell = copy.deepcopy(model)
	trainer = MyTrainer(
		model, loss, optimizer, args.output,
		checkpoint_shell=checkpoint_shell,
		lr_scheduler=lr_scheduler,
		device=torch.device('cuda'),
	)

	trainer.train_loop(train_loader, val_loader, start_epoch=start_epoch)


from SumoSupervisor import SumoSupervisor

import argparse
import os
import re
import shutil
import subprocess
import sys
import tempfile

def sumoImportError():
    sys.stderr.write("SUMO not found.\n")
    if sys.platform.startswith('linux'):
        if os.getenv('SNAP_NAME'):
            sys.stderr.write("Please set the 'SumoInterface.externController' field to TRUE and "
                             "launch the controller as extern controller.\n"
                             "When launching the controller as extern, you must specify the SUMO options like --no-netconvert "
                             "or --no-gui manually.\n"
                             "Install SUMO ")
        else:
            sys.stderr.write("Please install it ")
        sys.stderr.write("with `sudo apt install sumo sumo-tools` and reboot.\n")
    else:
        sys.stderr.write("Please install it following the instructions at https://sumo.dlr.de/docs/Installing/.\n")
    sys.exit("Or check that the SUMO_HOME environment variable points to the directory of your SUMO installation.")

# we need to import python modules from the $SUMO_HOME/tools directory
try:
    WEBOTS_HOME = os.path.normpath(os.environ.get('WEBOTS_HOME'))
    if 'SUMO_HOME' in os.environ:
        sumoPath = os.environ['SUMO_HOME']
        print('Using SUMO from %s' % sumoPath)
    else:
        sumoImportError()

    if sys.platform.startswith('darwin'):
        libraryVariablePath = 'DYLD_LIBRARY_PATH'
    elif sys.platform.startswith('linux'):
        libraryVariablePath = 'LD_LIBRARY_PATH'
    else:
        libraryVariablePath = 'PATH'
    path = os.environ.get(libraryVariablePath)
    addToPath = os.path.join(sumoPath, 'bin')
    if sys.platform.startswith('linux'):
        addToPath = os.path.join(WEBOTS_HOME, 'lib') + ';' + os.path.join(sumoPath, 'bin')
    if path is None:
        os.putenv(libraryVariablePath, addToPath)
    else:
        os.putenv(libraryVariablePath, addToPath + ';' + path)
    sys.path.append(os.path.join(sumoPath, 'tools'))
    import traci
    import sumolib
except ImportError:
    sumoImportError()

def get_options():
    """Parse the controler arguments."""
    parser = argparse.ArgumentParser()
    parser.add_argument("--no-gui", dest="noGUI", action="store_true", default=False,
                        help="runs the command line version of sumo")
    parser.add_argument("--verbose", dest="verbose", action="store_true", default=False,
                        help="prints sumo output in Webots console")
    parser.add_argument("--no-netconvert", dest="noNetconvert", action="store_true", default=False,
                        help="does not run netconvert before launching sumo")
    parser.add_argument("--disable-traffic-lights", dest="disableTrafficLights", action="store_true", default=False,
                        help="disables the update of the traffic lights state in Webots")
    parser.add_argument("--step", type=int, dest="step", default=200, help="specifies the time step of sumo [ms]")
    parser.add_argument("--max-vehicles", type=int, dest="maxVehicles", default=100,
                        help="specifies the maximum vehicles to add on Webots side")
    parser.add_argument("--rotate-wheels", dest="rotateWheels", action="store_true", default=False,
                        help="enables the wheels rotation.")
    parser.add_argument("--radius", type=int, dest="radius", default=-1,
                        help="specifies the visibility radius of the vehicles in meters (-1 means no limit)")
    parser.add_argument("--enable-height", dest="enableHeight", action="store_true", default=False,
                        help="specifies if height information should be extracted from the edge name")
    parser.add_argument("--directory", dest="directory", default="",
                        help="specifies the directory where are located the files defining the network")
    parser.add_argument("--port", type=int, dest="port", default=8873, help="specifies which port to use")
    parser.add_argument("--seed", type=int, dest="seed", default=1,
                        help="specifies the seed of the SUMO random number generator (0 for the '--random' option of SUMO)")
    parser.add_argument("--use-display", dest="useDisplay", action="store_true", default=False,
                        help="displays the gui view of SUMO in a Webots display (only working in gui mode)")
    parser.add_argument("--display-refresh-rate", type=int, dest="displayRefreshRate", default=1000,
                        help="specifies the refresh rate of the SUMO display in Webots")
    parser.add_argument("--display-zoom", type=float, dest="displayZoom", default=1.0,
                        help="specifies the initial zoom of the SUMO display in Webots (100 means no scaling)")
    parser.add_argument("--display-fit-size", dest="displayFitSize", action="store_true", default=False,
                        help="specifies if the image should be resized to fit the SUMO display size or not")
    parser.add_argument("--maximum-lateral-speed", type=float, dest="maximumLateralSpeed", default=2.5,
                        help="specifies the maximal lateral speed of any vehicle in meter per second.")
    parser.add_argument("--maximum-angular-speed", type=float, dest="maximumAngularSpeed", default=3,
                        help="specifies the maximal angular speed of any vehicle in radian per second.")
    parser.add_argument("--lane-change-delay", type=float, dest="laneChangeDelay", default=3,
                        help='specifies the time required to change lane (during this period position in Webots and SUMO may '
                        'not be perfectly synchronized anymore).')
    parser.add_argument("--sumo-arguments", dest="sumoArguments", default="", help="specifies additional SUMO arguments.")
    args = parser.parse_args()
    return args

# The main program starts from here

# Start sumo as a server and then connect and run
controller = SumoSupervisor()
options = get_options()

useDisplay = False
if options.noGUI:
    sumoBinary = os.path.join(sumoPath, 'bin', 'sumo')
else:
    if sys.platform.startswith('darwin') and not os.path.exists(os.path.join(os.sep, 'opt', 'X11')):
        sys.stderr.write("X11 is not installed and is required to launch the gui version of SUMO.\n")
        sys.stderr.write("You can easily install X11 following this link: https://support.apple.com/en-us/HT201341\n")
        sys.stderr.write("Starting command line version of SUMO instead.\n")
        sumoBinary = os.path.join(sumoPath, 'bin', 'sumo')
    else:
        sumoBinary = os.path.join(sumoPath, 'bin', 'sumo-gui')
        if options.useDisplay:
            useDisplay = True

# check if the target directory is in the WEBOTS_HOME path or not set, and adjust path if it is the case
directory = options.directory if options.directory == '' else os.path.normpath(options.directory)
if directory.startswith('WEBOTS_HOME'):
    directory = directory.replace('WEBOTS_HOME', WEBOTS_HOME)
elif directory == "":  # no directory set, use standard directory (same name of the world ending with '_net')
    directory = re.sub(r'.wbt$', '_net', controller.getWorldPath())
if not os.path.isdir(directory):
    sys.exit("You should specify in which directory are stored the network files associated to this world with the "
             "'--directory' argument or put them in the '%s' directory." % directory)

tmpDirectory = None
# generate the net file with the 'netconvert' utility
if not options.noNetconvert:
    # generate temporary directory and move network file in it
    tmpDirectory = tempfile.mkdtemp()
    for item in os.listdir(directory):
        s = os.path.join(directory, item)
        d = os.path.join(tmpDirectory, item)
        if os.path.isdir(s):
            shutil.copytree(s, d, True)
        else:
            shutil.copy2(s, d)
    directory = tmpDirectory
    print("Temporary network files generated in " + tmpDirectory + "\n")
    # check if default configuration file exist
    netConfigurationFile = os.path.join(directory, 'sumo.netccfg')
    if not os.path.isfile(netConfigurationFile):
        fileFound = ''
        # no default configuration file, try to find another one
        for file in os.listdir(directory):
            if file.endswith('.netccfg'):
                if fileFound == "":
                    netConfigurationFile = os.path.join(directory, file)
                    fileFound = file
                else:
                    print("More than one NETCONVERT configuration file found, using: " + fileFound + "\n")
                    break
    if not os.path.isfile(netConfigurationFile) and tmpDirectory is not None:
        shutil.rmtree(tmpDirectory)
        sys.exit("Could not find any NETCONVERT configuration file (*.netccfg).")
    if subprocess.call([os.path.join(sumoPath, 'bin', 'netconvert'), "-c", netConfigurationFile, "--xml-validation", "never"],
                       stdout=sys.stdout, stderr=sys.stderr) != 0:
        sys.exit("NETCONVERT failed to generate the network file.")

# this is the normal way of using traci. sumo is started as a
# subprocess and then the python script connects and runs
FNULL = open(os.devnull, 'w')
# check if default configuration file exist
configurationFile = os.path.join(directory, 'sumo.sumocfg')
if not os.path.isfile(configurationFile):
    fileFound = ""
    for file in os.listdir(directory):
        # no default configuration file, try to find another one
        if file.endswith('.sumocfg'):
            if fileFound == '':
                configurationFile = os.path.join(directory, file)
                fileFound = file
            else:
                print("More than one SUMO configuration file found, using: " + fileFound + "\n")
                break
if not os.path.isfile(configurationFile) and tmpDirectory is not None:
    shutil.rmtree(tmpDirectory)
    sys.exit("Could not find any SUMO configuration file (*.sumocfg).")

arguments = [sumoBinary, "-c", configurationFile, "--start",
             "--quit-on-end=true", "--step-length=" + str(options.step / 1000.0), "--remote-port", str(options.port)]

if options.seed == 0:
    arguments.append("--random")
else:
    arguments.append("--seed=" + str(options.seed))

if options.verbose:
    arguments.append("--verbose")

if os.path.isfile(os.path.join(directory, 'gui-settings.cfg')) and not options.noGUI:
    arguments.extend(["--gui-settings-file", os.path.join(directory, 'gui-settings.cfg')])

if options.sumoArguments != "":
    arguments.extend(options.sumoArguments.split())

sumoProcess = subprocess.Popen(arguments, stdout=FNULL, stderr=subprocess.STDOUT)
controller.run(options.port, options.disableTrafficLights, directory,
               options.step, options.rotateWheels, options.maxVehicles,
               options.radius, options.enableHeight, useDisplay,
               options.displayRefreshRate, options.displayZoom,
               options.displayFitSize, options.maximumLateralSpeed,
               options.maximumAngularSpeed, options.laneChangeDelay, traci, sumolib)
sumoProcess.terminate()

# remove temporary folder
if tmpDirectory is not None:
    print("Removing temporary network files in " + tmpDirectory + "\n")
    shutil.rmtree(tmpDirectory)


# -*- coding: utf-8 -*-
from PyQt5 import QtWidgets
from PyQt5 import QtGui
from PyQt5 import QtCore
from PyQt5.QtCore import Qt
from PyQt5.QtWidgets import *

# import Fuct_QThreadUI
from functools import partial
import urllib
import socket
import Fuct_Global

# 设置全局socket超时2秒
socket.setdefaulttimeout(4)

"""
其他窗口
"""
# 日线窗口
class DayLines(QtWidgets.QWidget):
    def __init__(self,code, parent = None):
        super(DayLines,self).__init__(parent)
        self.setWindowTitle(u'日线')
        self.days = 30
        self.Status = True
        icon = QtGui.QIcon()
        icon.addPixmap(QtGui.QPixmap(":/Imag/Imag/icon.ico"), QtGui.QIcon.Normal, QtGui.QIcon.Off)
        self.setWindowIcon(icon)
        self.setWindowFlags(Qt.Qt.WindowStaysOnTopHint|Qt.Qt.WindowMinimizeButtonHint)
        self.mainlayout = QGridLayout(self)
        self.mainlayout.setContentsMargins(0,0,0,0)
        self.myLabelEx = myLabel()
        self.mainlayout.addWidget(self.myLabelEx)
        self.File = "./data/tmp/Quote.png"
        if code[0] == "6":
            self.code = "0" + code
        else:
            self.code = "1" + code
        self.setPng()
        self.connect(self.myLabelEx, QtCore.SIGNAL('setpng'), self.CalcDays)

    def setPng(self):
        # 获取并设置图片
        url = "http://img1.money.126.net/chart/hs/kline/day/%d/%s.png" %(self.days,self.code)
        for i in range(5):
            content = urllib.urlopen(url).read()
            if len(content) > 10000:
                with open(self.File,"wb") as F:
                        F.write(content)
                self.myLabelEx.setPixmap(QPixmap(self.File))#####设置标签图片
                self.tmpCode = self.code
                self.tmpDays = self.days
                break

    def CalcDays(self):
        # 计算天数
        if self.days == 30:
            self.days = 90
        elif self.days == 90:
            self.days = 180
        else:
            self.days = 30
        self.setPng()

    def closeEvent(self, QCloseEvent):
        self.Status = False

# 重写QLabel，加入点击事件，用来显示日线数据
class myLabel(QLabel):
    def __init__(self, parent=None):
        super(myLabel, self).__init__(parent)

    def mousePressEvent(self, e):
        # 重载点击信号
        self.emit(QtCore.SIGNAL("setpng"))

# 龙虎榜日期控件
class dateWindow(QWidget):
    def __init__(self, parent=None):
        super(dateWindow, self).__init__(parent)
        self.setWindowTitle(u'日期查询')
        self.resize(300, 350)
        icon = QtGui.QIcon()
        icon.addPixmap(QtGui.QPixmap(":/Imag/Imag/icon.ico"), QtGui.QIcon.Normal, QtGui.QIcon.Off)
        self.setWindowIcon(icon)
        self.setWindowFlags(Qt.Qt.WindowStaysOnTopHint)
        self.cal = QtGui.QCalendarWidget(self)
        self.cal.setGridVisible(True)
        self.label = QtGui.QLabel(self)
        self.button = QtGui.QPushButton(self)
        self.button.setText(u"查   询")
        date = self.cal.selectedDate()
        self.label.setText(str(date.toPyDate()))
        vbox = QtGui.QVBoxLayout()
        vbox.addWidget(self.label)
        vbox.addWidget(self.cal)
        vbox.addWidget(self.button)
        self.setLayout(vbox)
        self.connect(self.cal, QtCore.SIGNAL('selectionChanged()'), self.showDate)
        self.button.clicked.connect(partial(self.button_clicked))

    def button_clicked(self):
        date = self.cal.selectedDate()
        date = str(date.toPyDate())
        self.emit(QtCore.SIGNAL("RankChice_dateWindow"), date)

    def showDate(self):
        date = self.cal.selectedDate()
        self.label.setText(str(date.toPyDate()))

# 涨停预测，自定义时间
class limitWindow(QWidget):
    def __init__(self, parent=None):
        super(limitWindow, self).__init__(parent)
        self.setWindowTitle(u'涨停预测')
        self.resize(300, 100)
        icon = QtGui.QIcon()
        icon.addPixmap(QtGui.QPixmap(":/Imag/Imag/icon.ico"), QtGui.QIcon.Normal, QtGui.QIcon.Off)
        self.setWindowIcon(icon)
        self.setWindowFlags(Qt.Qt.WindowStaysOnTopHint)
        # 定义控件
        self.label1 = QtGui.QLabel(self)
        self.label2 = QtGui.QLabel(self)
        self.edit = QtGui.QLineEdit()
        self.button = QtGui.QPushButton(self)
        # 设置控件
        self.label1.setText(u"*时间格式：2017-06-28 15:00")
        self.label1.setStyleSheet("color:rgb(255, 0, 0);")
        self.label2.setText(u"开始时间：")
        self.edit.setText(Fuct_Global.lastdayDateTime("%Y-%m-%d"+" 15:00"))
        self.button.setText(u"开始检索")

        vbox1 = QtGui.QVBoxLayout()
        hbox1 = QtGui.QHBoxLayout()

        hbox1.addWidget(self.label2)
        hbox1.addWidget(self.edit)

        vbox1.addWidget(self.label1)
        vbox1.addLayout(hbox1)
        vbox1.addWidget(self.button)
        self.setLayout(vbox1)
        self.button.clicked.connect(partial(self.button_clicked))

    def button_clicked(self):
        date = self.edit.text()
        date = date.replace("：",":").replace("  "," ").strip()
        self.close()
        self.emit(QtCore.SIGNAL("limit_dateWindow"), date)

# 涨跌幅追踪，自定义时间
class fuctuationWindow(QWidget):
    def __init__(self, parent=None):
        super(fuctuationWindow, self).__init__(parent)
        self.setWindowTitle(u'涨跌幅追踪')
        # self.resize(300, 100)
        icon = QtGui.QIcon()
        icon.addPixmap(QtGui.QPixmap(":/Imag/Imag/icon.ico"), QtGui.QIcon.Normal, QtGui.QIcon.Off)
        self.setWindowIcon(icon)
        self.setWindowFlags(Qt.Qt.WindowStaysOnTopHint)
        # 定义控件
        self.label1 = QtGui.QLabel(self)
        self.label2 = QtGui.QLabel(self)
        self.editStart = QtGui.QLineEdit()
        self.editEnd = QtGui.QLineEdit()
        self.QToolButtonStart = QtGui.QToolButton(self)
        self.QToolButtonEnd = QtGui.QToolButton(self)
        self.button = QtGui.QPushButton(self)
        # 定义控件
        day1 = QtGui.QPushButton(self)
        day1.setText(u"近一天")
        day2 = QtGui.QPushButton(self)
        day2.setText(u"近两天")
        day3 = QtGui.QPushButton(self)
        day3.setText(u"近三天")

        week1 = QtGui.QPushButton(self)
        week1.setText(u"近一周")
        week2 = QtGui.QPushButton(self)
        week2.setText(u"近两周")
        week3 = QtGui.QPushButton(self)
        week3.setText(u"近三周")

        month1 = QtGui.QPushButton(self)
        month1.setText(u"近一月")
        month2 = QtGui.QPushButton(self)
        month2.setText(u"近两月")
        month3 = QtGui.QPushButton(self)
        month3.setText(u"近三月")

        # 设置图标
        icon1 = QtGui.QIcon()
        icon1.addPixmap(QtGui.QPixmap(":/Imag/Imag/calendar.png"), QtGui.QIcon.Normal, QtGui.QIcon.Off)
        self.QToolButtonStart.setIcon(icon1)
        self.QToolButtonStart.setAutoRaise(True)
        self.QToolButtonEnd.setIcon(icon1)
        self.QToolButtonEnd.setAutoRaise(True)
        # 设置控件
        self.label1.setText(u"开始日期:")
        self.label2.setText(u"结束日期:")
        self.button.setText(u"开始检索")

        grid = QtGui.QGridLayout()
        grid.addWidget(day1, 0, 0)
        grid.addWidget(day2, 0, 1)
        grid.addWidget(day3, 0, 2)

        grid.addWidget(week1, 1, 0)
        grid.addWidget(week2, 1, 1)
        grid.addWidget(week3, 1, 2)

        grid.addWidget(month1, 2, 0)
        grid.addWidget(month2, 2, 1)
        grid.addWidget(month3, 2, 2)

        vbox1 = QtGui.QVBoxLayout()
        hbox1 = QtGui.QHBoxLayout()
        hbox2 = QtGui.QHBoxLayout()

        hbox1.addWidget(self.label1)
        hbox1.addWidget(self.editStart)
        hbox1.addWidget(self.QToolButtonStart)

        hbox2.addWidget(self.label2)
        hbox2.addWidget(self.editEnd)
        hbox2.addWidget(self.QToolButtonEnd)

        vbox1.addLayout(grid)
        vbox1.addLayout(hbox1)
        vbox1.addLayout(hbox2)
        vbox1.addWidget(self.button)
        self.setLayout(vbox1)
        self.button.clicked.connect(partial(self.button_clicked))

    def button_clicked(self):
        start = self.editStart.text()
        end = self.editEnd.text()
        self.emit(QtCore.SIGNAL("fuctuation_dateWindow"), start, end)

if __name__ == '__main__':
    pass


import string
from abc import ABC, abstractmethod
from math import log
from typing import List, Tuple, Generator

import numpy as np
from nltk.corpus import stopwords
from sklearn.metrics.pairwise import cosine_similarity
from util import tqdm

from util import Entity, Sample, get_question
from nltk import word_tokenize, SnowballStemmer, defaultdict
import ailog
import logging

ailog.setup_logging('logging.conf')
logger = logging.getLogger("This")

stemmer = SnowballStemmer('english')
stop_words = stopwords.words('english')

class Scorer(ABC):
    def __init__(self, remove_punctuation=True, remove_stopwords=False,
                 do_stem=False):
        self.remove_punctuation = remove_punctuation
        self.remove_stopwords = remove_stopwords
        self.do_stem = do_stem
        self.c = Curry()
        if self.remove_punctuation:
            self.c += unpunct
        if self.remove_stopwords:
            self.c += unstop
        if self.remove_punctuation:
            self.c += unpunct

    @abstractmethod
    def __call__(self, sentence, other_sentence, rest_sentences,
                 **kwargs) -> int:
        ...

class Curry:
    def __init__(self, function=None):
        if function:
            self.functions = [function]
        else:
            self.functions = []

    def __add__(self, other):
        c = Curry()
        c.functions = self.functions
        c.functions += [other]
        return c

    def __call__(self, arg):
        for f in self.functions:
            arg = f(arg)
        return arg

def fuzzy_substr(s1: str, s2: str):
    return s1 == s2

def remove_punctuation(s: str, trailing_only=False):
    if trailing_only:
        i = len(s) - 1
        while i > 0 and s[i] in string.punctuation:
            i -= 1
        return s[:i]
    return "".join(c for c in s if c not in string.punctuation)

def fuzzy_subsentence(sentence: List[str], other_sentence: List[str]):
    tries = len(other_sentence) - len(sentence) + 1
    if tries < 0:
        logger.debug(f'{sentence} in {other_sentence}: False')
        return False
    for i in range(tries):
        if all(fuzzy_substr(s1, s2) for s1, s2 in
               zip(sentence, other_sentence[i:])):
            logger.debug(f'{sentence} in {other_sentence}: True')
            return True
        logger.debug(f'{sentence} in {other_sentence}: False')
    return False

def stem(sentence: List[str]) -> Generator[str, None, None]:
    return (stemmer.stem(w) for w in sentence)

def unstop(sentence: List[str]) -> Generator[str, None, None]:
    return (w for w in sentence if w not in stop_words)

def unpunct(sentence: List[str]) -> Generator[str, None, None]:
    return (w for w in sentence if w not in string.punctuation)

def normalize(sentence: List[str]) -> List[str]:
    return [t.lower().strip() for t in sentence]

class MaxNgramScorer(Scorer):

    def __call__(self, sentence: List[str], other_sentence: List[str], *args,
                 **kwargs):

        if self.c.functions:
            sentence = list(self.c(sentence))
            other_sentence = list(self.c(other_sentence))

        max_ngram = 0
        sentence_idx = 0
        for i, _ in enumerate(sentence):
            subsentence = sentence[i:]
            for j, _ in enumerate(subsentence, 1):
                if fuzzy_subsentence(subsentence[:j],
                                     other_sentence) and j > max_ngram:
                    max_ngram = len(subsentence[:j])
                    sentence_idx = i
        return max_ngram

class GloveDistanceScorer(Scorer):
    def __init__(self, glove_file: str, m):
        super().__init__(do_stem=False)
        self.m = m
        self.words = dict()
        with open(glove_file, 'r') as f:
            content = f.readlines()
        for line in tqdm(content):
            word, embedding = line.split(" ", 1)
            embedding = np.array([float(val) for val in embedding.split(" ")])
            self.words[word] = embedding

    def w2v(self, word):
        try:
            return self.words[word].reshape(1, -1)
        except KeyError:
            return np.array([0] * 300, dtype=float).reshape(1, -1)

    def __call__(self, sentence: List[str], other_sentence: List[str], *args,
                 **kwargs):
        if self.c.functions:
            sentence = list(self.c(sentence))
            other_sentence = list(self.c(other_sentence))
        sims = np.array(
            [cosine_similarity(self.w2v(w), self.w2v(ow))
             for w in sentence
             for ow in other_sentence]
        )
        result = sims[sims.argsort()[-self.m:]].mean()
        return 0.0 if np.isnan(result) else float(result)

class MaxContainsScorer(Scorer):
    def __init__(self, remove_punctuation=True, remove_stopwords=True,
                 do_stem=True):
        super().__init__(remove_punctuation, remove_stopwords, do_stem)

    def __call__(self, sentence: List[str], other_sentence: List[str], *args,
                 **kwargs):
        if self.c.functions:
            sentence = list(self.c(sentence))
            other_sentence = list(self.c(other_sentence))
        return sum(1 for w in sentence if w in other_sentence)

ANY = 'all'
FULL_ONLY = 'full'

def yield_ngrams(sentence, n=1, skip_punctuation=True,
                 skip_stopwords=FULL_ONLY):
    i = 0
    if skip_punctuation:
        sentence = [s for s in sentence if s not in string.punctuation]
    if skip_stopwords == ANY:
        sentence = [s for s in sentence if s not in stopwords.words('english')]
    while i + n <= len(sentence):
        ngram = sentence[i:i + n]
        if all(w in stopwords.words('english') for w in ngram):
            pass
        else:
            yield ngram
        i += 1

class ContainsUniqueNgramScorer(Scorer):
    def __init__(self, remove_punctuation=True, remove_stopwords=False,
                 do_stem=True, n=2):

        super().__init__(remove_punctuation, remove_stopwords, do_stem)
        self.n = n

    def __call__(self, sentence, other_sentence, rest_sentences, *args,
                 **kwargs):
        if self.c.functions:
            sentence = list(self.c(sentence))
            other_sentence = list(self.c(other_sentence))

        for n_gram in yield_ngrams(sentence, self.n):
            if fuzzy_subsentence(n_gram, other_sentence) and not any(
                    fuzzy_subsentence(n_gram, s) for s in rest_sentences):
                return True
        return False

def avg_tf_idf(question, supporting_facts, rest):
    tf_idf_scores = tf_idf(question, supporting_facts, rest)
    sf_scores = defaultdict(int)
    rest_scores = defaultdict(int)
    for w, scores in tf_idf_scores.items():
        for i, score in enumerate(scores[0]):
            sf_scores[i] += score
        for i, score in enumerate(scores[1]):
            rest_scores[i] += score
    num_query_terms = len(tf_idf_scores.keys())
    for i, s in sf_scores.items():
        sf_scores[i] = s / num_query_terms

    for i, s in rest_scores.items():
        rest_scores[i] = s / num_query_terms
    return sf_scores, rest_scores

def tf_idf(question, supporting_facts, rest):
    c = Curry(stem) + Curry(unpunct) + Curry(unstop)
    question = list(c(question))
    for i, sf in enumerate(supporting_facts):
        supporting_facts[i] = list(c(sf))

    for i, r in enumerate(rest):
        rest[i] = list(c(r))
    tf_idf_scores = dict()
    for w in question:
        tf_supporting_facts = []
        for sf in supporting_facts:
            # sf = list(c(sf))

            tf_supporting_facts.append(1 / len(sf) if w in sf else 0)
            logger.debug(tf_supporting_facts)
        tf_rest = []
        for r in rest:
            # r = list(c(r))

            tf_rest.append(1 / len(sf) if w in r else 0)
        n = len(supporting_facts + rest)
        d = (1 + sum(tf_supporting_facts) + sum(tf_rest))

        idf = log(n / d)
        tf_idf_scores[w] = (
            [tf * idf for tf in tf_supporting_facts],
            [tf * idf for tf in tf_rest]
        )

    return tf_idf_scores

def ratio(question, supporting_fact, rest, operation: Scorer, aggregate=max):
    sf_score = operation(question, supporting_fact, rest)
    results = []
    for i, r in enumerate(rest):
        new_rest = rest[:] + supporting_fact
        new_rest.remove(r)
        result = operation(question, r, new_rest)
        results.append(result)
    rest_score = aggregate(results)
    epsilon = 0.5
    return (epsilon + sf_score) / (rest_score + epsilon)

def filter(question, supporting_facts, rest, operation: Scorer, threshold):
    everything = supporting_facts + rest
    result = []
    for s in everything:
        new_rest = rest[:]
        new_rest.remove(s)
        score = operation(question, s, rest)
        if score > threshold:
            result.append((score, s))
    return result

def average_precision(solution: List[List[str]], gold: List[List[str]]):
    subset_result = solution[:len(gold)]
    logger.debug(subset_result)
    # return sum(1 for s in subset_result if s in gold) / len(gold)
    return sum(sum(1 for s in subset_result[:i] if s in gold) / i for i, _ in
               enumerate(subset_result, 1)) / len(gold)

def last_correct_rank(solution: List, gold: List):
    for i, candidate in enumerate(solution, 1):
        if candidate in gold:
            gold.remove(candidate)
            if not gold:
                return i
    return len(solution)

def rank(question, supporting_facts, rest, operation: Scorer,
         eval_function) -> float:
    candidates = rest + supporting_facts

    def key(candidate):
        logger.debug(candidate)
        rest = candidates[:]
        rest.remove(candidate)
        return operation(question, candidate, rest)

    result = sorted(candidates, key=key, reverse=True)
    logger.debug(result)
    return eval_function(result, supporting_facts)

def fuzzy_map_supporting_fact(supporting_fact: str,
                              sentences: List[str]) -> int:
    for i, sentence in enumerate(sentences):
        if remove_punctuation(supporting_fact).strip() == remove_punctuation(
                sentence).strip():
            return i
    raise ValueError(
        "No supporting fact mapped! {}\n{}".format(supporting_fact,
                                                   '\n'.join(sentences)))

def get_supporting_fact(annotations: List[Entity]) -> List[str]:
    return [e.surface_form for e in annotations if e.type == "SupportingFact"]

def prepare(sample: Sample, annotations: List[Entity], paragraph_extract,
            just_supp_fact_ids=False):
    supp_facts_annotations = get_supporting_fact(annotations)
    sentences = paragraph_extract(sample.raw_text)
    question = get_question(sample.raw_text)
    try:
        supp_fact_ids = [fuzzy_map_supporting_fact(ann, sentences) for ann in
                         supp_facts_annotations]
    except ValueError as e:
        if len(supp_facts_annotations) == 1:
            sf = supp_facts_annotations[0]
            try:
                fuzzy_map_supporting_fact(sf, [question])
                supp_fact_ids = []
            except ValueError:
                raise e
        else:
            raise e

    if not just_supp_fact_ids:
        supp_fact_ids = sorted(supp_fact_ids, reverse=True)

        supp_facts = [sentences.pop(i) for i in supp_fact_ids]

        return (word_tokenize(question),
                [word_tokenize(s) for s in supp_facts],
                [word_tokenize(s) for s in sentences])
    else:
        return (word_tokenize(question),
                supp_fact_ids,
                [word_tokenize(s) for s in sentences])

def run_ratio_on_sample(samples: List[Tuple[Sample, List[Entity]]],
                        paragraph_extract, operation, aggregate=max):
    results = []
    for i, s in enumerate(tqdm(samples)):
        try:
            q, sf, sents = prepare(*s, paragraph_extract)
        except ValueError as e:
            raise ValueError(f"{i}: {str(e)}")
        if not sf:
            continue
        micro_results = [ratio(q, ssf, sents, operation, aggregate) for ssf in
                         sf]
        # if not micro_results:
        results.append(sum(micro_results) / len(micro_results))
    return results

def run_rank_on_sample(samples: List[Tuple[Sample, List[Entity]]],
                       paragraph_extract, operation,
                       eval_function):
    results = []
    for i, s in enumerate(samples):
        try:
            q, sf, sents = prepare(*s, paragraph_extract)
        except ValueError as e:
            raise ValueError(f"{i}: {str(e)}")
        if sf:
            result = rank(q, sf, sents, operation, eval_function)
            results.append(result)
    return results


#!/usr/bin/env python3
import argparse
import json
import multiprocessing
import os
import re
import shlex
import shutil
import subprocess
import sys
import time

# noinspection PyBroadException
try:
    import mutagen
except:
    mutagen = None

# Your unique announce URL
announce = ''

# Where to output .torrent files
torrent_output = '.'

# Where to save transcoded albums
transcode_output = '.'

# The default formats to transcode to
default_formats = '320,v0'

# Whether or not to transcode by default
default_transcode = True

# Whether or not to make .torrent files by default
default_torrent = True

# The maximum number of threads to maintain. Any number less than 1 means the
# script will use the number of CPU cores in the system. This is the default
# value for the -c (--cores) option.
max_threads = 0

# I prefix torrents I download as FL for Freeleech, UL for Upload, etc. Any
# prefix in this set will be removed from any transcoded albums and from the
# resulting torrent files created.
ignored_prefixes = {
}

# torrent_commands is the set of all ways to create a torrent using various
# torrent clients. These are the following replacements:
# {0}: Source directory to create a torrent from
# {1}: Output .torrent file
# {2}: Your announce URL
torrent_commands = {
    'transmission-create -p -o {1} -t {2} {0}',
    'mktorrent -p -o {1} -a {2} {0}'
}
torrent_command = None

# transcode_commands is the map of how to transcode into each format. The
# replacements are as follows:
# {0}: The input file (*.flac)
# {1}: The output file (*.mp3 or *.m4a)
# {2}: Song title
# {3}: Artist
# {4}: Album
# {5}: date
# {6}: track number
ffmpeg = 'ffmpeg -threads 1 '
transcode_commands = {
    '16-48': ffmpeg + '-i {0} -acodec flac -sample_fmt s16 -ar 48000 {1}',
    '16-44': ffmpeg + '-i {0} -acodec flac -sample_fmt s16 -ar 44100 {1}',
    'alac': ffmpeg + '-i {0} -acodec alac {1}',
    '320': ffmpeg + '-i {0} -acodec libmp3lame -ab 320k {1}',
    'v0': 'flac --decode --stdout {0} | lame -V 0 -q 0 --add-id3v2 --tt {2} --ta {3} --tl {4} --ty {5} --tn {6} - {1}',
    'v1': 'flac --decode --stdout {0} | lame -V 1 -q 0 --add-id3v2 --tt {2} --ta {3} --tl {4} --ty {5} --tn {6} - {1}',
    'v2': 'flac --decode --stdout {0} | lame -V 2 -q 0 --add-id3v2 --tt {2} --ta {3} --tl {4} --ty {5} --tn {6} - {1}'
}

# extensions maps each codec type to the extension it should use
extensions = {
    '16-48': 'flac',
    '16-44': 'flac',
    'alac': 'm4a',
    '320': 'mp3',
    'v0': 'mp3',
    'v2': 'mp3'
}

# codecs is use in string matching. If, in naming an album's folder name, you
# would use [FLAC] or [ALAC] or [320], then the lowercase contents of the
# brackets belongs in codecs so it can be matched and replaced with the
# transcode codec type.
codecs = {
    'wav',
    'flac', 'flac 24bit', 'flac 16-44', 'flac 16-48', 'flac 24-44', 'flac 24-48', 'flac 24-96', 'flac 24-196',
    '16-44', '16-48', '24-44', '24-48', '24-96', '24-196',
    'alac',
    '320', '256', '224', '192',
    'v0', 'apx', '256 vbr', 'v1', '224 vbr', 'v2', 'aps', '192 vbr'
}

# The list of lossless file extensions. While m4a can be lossy, it's up to you,
# the user, to ensure you're only transcoding from a lossless source material.
LOSSLESS_EXT = {'flac', 'wav', 'm4a'}

# The list of lossy file extensions
LOSSY_EXT = {'mp3', 'aac', 'opus', 'ogg', 'vorbis'}

# The version number
__version__ = '0.7'

exit_code = 0
FILE_NOT_FOUND = 1 << 0
ARG_NOT_DIRECTORY = 1 << 1
NO_TORRENT_CLIENT = 1 << 2
TRANSCODE_AGAINST_RULES = 1 << 3
TRANSCODE_DIR_EXISTS = 1 << 4
UNKNOWN_TRANSCODE = 1 << 5
NO_ANNOUNCE_URL = 1 << 6
NO_TRANSCODER = 1 << 7
TORRENT_ERROR = 1 << 8
TRANSCODE_ERROR = 1 << 9

def enumerate_contents(directory):
    has_lossy = False
    lossless_files = []
    data_files = []
    directories = []

    for root, _, files in os.walk(directory):
        root = root[len(directory):].lstrip('/')

        if len(root) > 0:
            directories.append(root)

        for file in files:
            extension = file[file.rfind('.') + 1:]
            if len(root) > 0:
                file = root + '/' + file

            if extension in LOSSLESS_EXT:
                lossless_files.append(file)
            else:
                if extension in LOSSY_EXT:
                    has_lossy = True
                data_files.append(file)

    return directories, data_files, has_lossy, lossless_files

def format_command(command, *args):
    safe_args = [quote(arg) for arg in args]
    return command.format(*safe_args)

def command_exists(command):
    return which(shlex.split(command)[0]) is not None

def find_torrent_command(commands):
    for command in commands:
        if command_exists(command):
            return command

    return None

def to_str(data):
    if type(data) is str:
        return to_str(data.encode('utf-8', 'surrogateescape'))
    else:
        return data.decode('utf-8', 'ignore')

def copy_contents(src, dst, dirs, files):
    os.mkdir(dst)

    for subdir in dirs:
        os.mkdir(dst + '/' + subdir)

    for file in files:
        shutil.copy(src + '/' + file, dst + '/' + file)

def make_torrent(directory, output, announce_url):
    global torrent_command, exit_code
    print('Making torrent for ' + directory)

    if torrent_command is None:
        torrent_command = find_torrent_command(torrent_commands)
        if torrent_command is None:
            print('No torrent client found, can\'t create a torrent')
            exit_code |= NO_TORRENT_CLIENT
            return

    command = format_command(torrent_command, directory, torrent_output + '/' + output, announce_url)
    torrent_status = subprocess.call(command, shell=True)
    if torrent_status != 0:
        print('Making torrent file exited with status {}!'.format(torrent_status))
        exit_code |= TORRENT_ERROR

def get_tags(filename):
    command = 'ffprobe -v 0 -print_format json -show_format'.split(' ') + [filename]
    info = json.loads(to_str(subprocess.Popen(command, stdout=subprocess.PIPE).communicate()[0]))

    if 'format' not in info or 'tags' not in info['format']:
        return '', '', '', '', ''

    tags = info['format']['tags']
    tags = {key.lower(): tags[key] for key in tags}
    parsed = {'title': '', 'artist': '', 'album': '', 'date': '', 'track': ''}

    for key in tags:
        if key in parsed:
            parsed[key] = tags[key]

    if len(parsed['track']) > 0 and 'tracktotal' in tags and len(tags['tracktotal']) > 0:
        parsed['track'] += '/' + tags['tracktotal']

    return parsed['title'], parsed['artist'], parsed['album'], parsed['date'], parsed['track']

def copy_album_art(source, dest):
    if mutagen is None:
        return

    flac = mutagen.File(source)

    if len(flac.pictures) > 0:
        # noinspection PyUnresolvedReferences
        apic = mutagen.id3.APIC(mime=flac.pictures[0].mime, data=flac.pictures[0].data)

        mp3 = mutagen.File(dest)
        mp3.tags.add(apic)
        mp3.save()

# noinspection PyUnresolvedReferences
def transcode_files(src, dst, files, command, extension):
    global exit_code
    remaining = files[:]
    transcoded = []
    threads = [None] * max_threads
    filenames = []

    transcoding = True

    while transcoding:
        transcoding = False

        for i in range(len(threads)):
            if threads[i] is None or threads[i].poll() is not None:
                if threads[i] is not None:
                    if threads[i].poll() != 0:
                        print('Error transcoding, process exited with code {}'.format(threads[i].poll()))
                        print('stderr output...')
                        print(to_str(threads[i].communicate()[1]))
                    # noinspection PyBroadException
                    try:
                        threads[i].kill()
                    except Exception as _:
                        pass

                threads[i] = None

                if len(remaining) > 0:
                    transcoding = True
                    file = remaining.pop()
                    transcoded.append(dst + '/' + file[:file.rfind('.') + 1] + extension)
                    threads[i] = subprocess.Popen(
                        format_command(command, src + '/' + file, transcoded[-1], *get_tags(src + '/' + file)),
                        stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True,
                        universal_newlines=True
                    )
                    filenames.append((src + '/' + file, transcoded[-1]))
                    print(to_str('Transcoding {} ({} remaining)'.format(file, len(remaining))))
            else:
                transcoding = True

        time.sleep(0.05)

    for file in transcoded:
        if not os.path.isfile(file):
            print('An error occurred and {} was not created'.format(file))
            exit_code |= TRANSCODE_ERROR
        elif os.path.getsize(file) == 0:
            print('An error occurred and {} is empty'.format(file))
            exit_code |= TRANSCODE_ERROR

    try:
        for pair in filenames:
            copy_album_art(*pair)
    except:
        pass

def transcode_album(source, directories, files, lossless_files, formats, explicit_transcode, mktorrent):
    global exit_code

    codec_regex = r'\[(' + '|'.join([codec for codec in codecs]) + r')\](?!.*\/.*)'
    dir_has_codec = re.search(codec_regex, source, flags=re.IGNORECASE) is not None

    for transcode_format in formats:
        if not command_exists(transcode_commands[transcode_format]):
            command = shlex.split(transcode_commands[transcode_format])[0]
            print('Cannot transcode to ' + transcode_format + ', "' + command + '" not found')
            exit_code |= NO_TRANSCODER
            continue

        print('\nTranscoding to ' + transcode_format)

        if dir_has_codec:
            transcoded = re.sub(codec_regex, '[{}]'.format(transcode_format.upper()), source, flags=re.IGNORECASE)
        else:
            transcoded = source.rstrip() + ' [{}]'.format(transcode_format.upper())

        transcoded = transcoded[transcoded.rfind('/') + 1:]

        for prefix in ignored_prefixes:
            if transcoded.startswith(prefix):
                transcoded = transcoded[len(prefix):]
                break

        transcoded = transcode_output + '/' + transcoded

        if os.path.exists(transcoded):
            if explicit_transcode:
                exit_code |= TRANSCODE_DIR_EXISTS

            print('Directory already exists: ' + transcoded)
            continue

        copy_contents(source, transcoded, directories, files)
        transcode_files(source, transcoded, lossless_files, transcode_commands[transcode_format],
                        extensions[transcode_format])

        if mktorrent:
            make_torrent(transcoded, transcoded[transcoded.rfind('/'):] + '.torrent', announce)

def is_transcode_allowed(has_lossy, lossless_files, explicit_transcode):
    global exit_code

    if has_lossy > 0:
        if len(lossless_files) == 0:
            print('Cannot transcode lossy formats, exiting')
            exit_code |= TRANSCODE_AGAINST_RULES
            return False
        elif not explicit_transcode:
            print('Found mixed lossy and lossless, you must explicitly enable transcoding')
            exit_code |= TRANSCODE_AGAINST_RULES
            return False

    if len(lossless_files) == 0:
        print('Nothing to transcode!')
        exit_code |= TRANSCODE_AGAINST_RULES
        return False

    return True

def check_main_args(directory, transcode_formats, explicit_torrent):
    global exit_code
    code = 0

    if not os.path.exists(directory):
        print('The directory "{}" doesn\'t exist'.format(directory))
        code |= FILE_NOT_FOUND
    elif os.path.isfile(directory):
        print('The file "{}" is not a directory'.format(directory))
        code |= ARG_NOT_DIRECTORY

    for i in range(len(transcode_formats)):
        transcode_formats[i] = transcode_formats[i].lower()

        if transcode_formats[i] not in transcode_commands.keys():
            print('No way of transcoding to ' + transcode_formats[i])
            code |= UNKNOWN_TRANSCODE

    if explicit_torrent and (announce is None or len(announce) == 0):
        print('You cannot create torrents without first setting your announce URL')
        code |= NO_ANNOUNCE_URL

    exit_code |= code

    return code == 0

def process_album(directory, do_transcode, explicit_transcode, transcode_formats, do_torrent, explicit_torrent,
                  original_torrent):
    global exit_code
    directory = os.path.abspath(directory)

    if not (check_main_args(directory, transcode_formats, explicit_torrent)):
        return

    if original_torrent:
        make_torrent(directory, directory[directory.rfind('/'):] + '.torrent', announce)

    if do_transcode:
        directories, data_files, has_lossy, lossless_files = enumerate_contents(directory)

        if is_transcode_allowed(has_lossy, lossless_files, explicit_transcode):
            transcode_album(directory, directories, data_files, lossless_files, transcode_formats, explicit_transcode,
                            do_torrent)

def parse_args():
    description = '(Version {}) Transcode albums and create torrents in one command. Default behavior can be changed ' \
                  'by opening %(prog)s with a text editor and changing the variables at the top of the file.' \
        .format(__version__)

    parser = argparse.ArgumentParser(description=description)
    transcode_group = parser.add_mutually_exclusive_group()
    torrent_group = parser.add_mutually_exclusive_group()

    parser.add_argument('album', help='The album to process', nargs='+')

    parser.add_argument('-v', '--version', action='version', version='%(prog)s ' + __version__)

    announce_postfix = ' (Usable URL set)' if len(announce) > 0 else ''
    parser.add_argument('-a', '--announce', action='store', default=announce,
                        help='The torrent announce URL to use' + announce_postfix)

    postfixes = {
        't': ' (default)' if default_transcode else '',
        'T': ' (default)' if not default_transcode else '',
        'm': ' (default)' if default_torrent else '',
        'M': ' (default)' if not default_torrent else ''
    }
    transcode_group.add_argument('-t', '--transcode', action='store_true',
                                 help='Transcode the given album into other formats' + postfixes['t'])
    transcode_group.add_argument('-T', '--no-transcode', action='store_true',
                                 help='Ensures the given album is NOT transcoded' + postfixes['T'])

    torrent_group.add_argument('-m', '--make-torrent', action='count', default=0,
                               help='Creates a torrent of any transcoded albums. Specify more than once to also create '
                                    'a torrent of the source album (e.g. -mm).' + postfixes['m'])
    torrent_group.add_argument('-M', '--no-torrent', action='store_true',
                               help='Ensures no .torrent files are created' + postfixes['M'])

    parser.add_argument('-f', '--formats', action='store', default=default_formats,
                        help='The comma-separated formats to transcode to (can be of 16-48,16-44,alac,320,v0,v1,v2) '
                             '(default: %(default)s)')
    parser.add_argument('-c', '--cores', action='store', type=int, default=max_threads,
                        help='The number of cores to transcode on. Any number below 1 means to use the '
                             'number of CPU cores in the system (default: %(default)s)')

    parser.add_argument('-o', '--torrent-output', action='store', default=torrent_output,
                        help='The directory to store any created .torrent files (default: %(default)s)')
    parser.add_argument('-O', '--transcode-output', action='store', default=transcode_output,
                        help='The directory to store any transcoded albums in (default: %(default)s)')

    return parser.parse_args()

def main(args):
    global exit_code, announce, torrent_output, transcode_output, max_threads

    announce = args.announce

    do_transcode = default_transcode and not args.no_transcode
    explicit_transcode = args.transcode
    formats = args.formats.split(',')

    do_torrent = default_torrent and not args.no_torrent
    explicit_torrent = args.make_torrent
    original_torrent = args.make_torrent == 2

    if not explicit_torrent and len(announce) == 0:
        do_torrent = False
    if mutagen is None and 'v0' in formats:
        print('Mutagen is not installed; album art won\'t be copied to VBR transcodes')
        print('To keep album art, install mutagen (try "sudo python3 -m pip install mutagen")')
        if sys.version_info[1] < 4:
            print('Your python version is <3.4, you must install pip yourself before mutagen.')

    torrent_output = args.torrent_output
    transcode_output = args.transcode_output

    max_threads = args.cores
    if max_threads < 1:
        max_threads = multiprocessing.cpu_count()

    if not os.path.isdir(torrent_output):
        print('The given torrent output dir ({}) is not a directory'.format(torrent_output))
        exit_code |= ARG_NOT_DIRECTORY
    elif not os.path.isdir(transcode_output):
        print('The given transcode output dir ({}) is not a directory'.format(transcode_output))
        exit_code |= ARG_NOT_DIRECTORY

    if exit_code != 0:
        return

    first_print = True

    for album in args.album:
        if not first_print:
            print('\n\n')
        first_print = False

        print('Processing ' + album)
        process_album(album, do_transcode, explicit_transcode, formats, do_torrent, explicit_torrent, original_torrent)

#
# The following functions are copied for use in older version of Python. They
# are standard library functions in Python >3.2 that don't exist in 3.2 itself.
#
_find_unsafe = re.compile(r'[^\w@%+=:,./-]', re.ASCII).search

def quote(s):
    """Return a shell-escaped version of the string *s*."""
    if not s:
        return "''"
    if _find_unsafe(s) is None:
        return s

    return "'" + s.replace("'", "'\"'\"'") + "'"

def which(cmd, mode=os.F_OK | os.X_OK, path=None):
    def _access_check(fn, _mode):
        return (os.path.exists(fn) and os.access(fn, _mode)
                and not os.path.isdir(fn))

    if os.path.dirname(cmd):
        if _access_check(cmd, mode):
            return cmd
        return None

    if path is None:
        path = os.environ.get("PATH", os.defpath)
    if not path:
        return None
    path = path.split(os.pathsep)

    if sys.platform == "win32":
        if os.curdir not in path:
            path.insert(0, os.curdir)

        pathext = os.environ.get("PATHEXT", "").split(os.pathsep)
        if any(cmd.lower().endswith(ext.lower()) for ext in pathext):
            files = [cmd]
        else:
            files = [cmd + ext for ext in pathext]
    else:
        files = [cmd]

    seen = set()
    for _dir in path:
        normdir = os.path.normcase(_dir)
        if normdir not in seen:
            seen.add(normdir)
            for thefile in files:
                name = os.path.join(_dir, thefile)
                if _access_check(name, mode):
                    return name
    return None

main(parse_args())
if exit_code != 0:
    print('An error occurred, exiting with code {}'.format(exit_code))
sys.exit(exit_code)


import re
import requests
# # NOTE: consider switching to HTTPConnection as per https://stackoverflow.com/questions/16778435/python-check-if-website-exists for head requests
# from http.client import HTTPConnection

# matcher for valid route protocol
url_string = r'https://\S+|http://\S+'
url_matcher = re.compile(url_string)

def parsable(url:str):
    ''' Deterimines if url has valid protocol '''
    return True if url_matcher.fullmatch(url) else False

def valid(url:str, timeout:int=2):
    ''' Determines if url is valid by requesting head '''
    try:
        request = requests.get(url, timeout=timeout)
        return (request.status_code==200)
    except Exception as e:
        return False

def fix_url(url:str):
    ''' Adds proper route to url. Returns url if fixed else return False '''
    if not parsable(url):
        if url.startswith('http'):
            pass
        elif url.startswith('www'):
            url = f'https://{url}'
        else:
            url = f'https://{url}'
    # try connecting to url to validate
    return (url if valid(url) else False)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models

class Migration(migrations.Migration):

    dependencies = [
        ('Finance', '0004_remove_paymentorder_payable'),
    ]

    operations = [
        migrations.AlterField(
            model_name='paymentorder',
            name='shipment_order',
            field=models.ForeignKey(verbose_name='订单', on_delete=models.SET('订单被删除'), to='ShipmentOrder.ShipmentOrder'),
        ),
    ]