import json
import re
import pandas as pd
import csv
import os

seperator = '|'
#file_cnt = 561
pkt_dir = "./pcap_data/"

#read files from dir
def iter_dir():
    path = ""

def find_mac_obj(json_root_arr, mac):
    for json_pkt_obj in json_root_arr:
        if(json_pkt_obj['mac'] == mac):
            return json_pkt_obj
    #create mac_obj and add to json_root_arr
    json_pkt_obj = {}
    json_pkt_obj['mac'] = mac
    json_root_arr.append(json_pkt_obj)
    return json_pkt_obj

def extract_ptcl(layer_obj):
    frame_obj = layer_obj['frame']
    ptcl_str = frame_obj['frame.protocols']
    ptcl = ptcl_str.split(':')[-1]
    if ('data' == ptcl) and ('udp' in layer_obj):
        dst_port = layer_obj['udp']['udp.dstport']
        ptcl = 'udp_' + dst_port
    elif ('data' == ptcl):
        ptcl = ptcl_str.split(':')[-2]
    return ptcl

def gene_ptcl_obj(json_pkt_obj, ptcl):
    if 'protocol' in json_pkt_obj:
        ptcl_str = json_pkt_obj['protocol']
        json_pkt_obj['protocol'] = ptcl_str + ',' + ptcl
    else:
        json_pkt_obj['protocol'] = ptcl

#2 layer:     "ssdp":{...}
def add_ptcl_obj(json_pkt_obj, ptcl):
    if ptcl in json_pkt_obj:
        return
    else:
        ptcl_obj = {}
        json_pkt_obj[ptcl] = ptcl_obj

#suit for llmnr/nbns/smb
def add_ptcl_val(json_pkt_obj, ptcl, val):
    if ptcl in json_pkt_obj:
        ptcl_str = json_pkt_obj[ptcl]
        if(-1 == ptcl_str.find(val)):
            json_pkt_obj[ptcl] = ptcl_str + '_nextpacket_' + val
    else:
        json_pkt_obj[ptcl] = val

def add_mdns_rrs(layer_obj, key):
    val = ''
    if type(layer_obj['mdns'][key]) == str:
        if ('' != layer_obj['mdns'][key]):
            val = layer_obj['mdns'][key]
        return val
    keys = layer_obj['mdns'][key].keys()
    model = ''
    for rr_key in keys:
        if not val:
            val = rr_key.replace(':', seperator).replace('.', seperator).replace(',', seperator)
        else:
            val = val + '_nextrr_' + rr_key.replace(':', seperator).replace('.', seperator).replace(',', seperator)

        if 'dns.resp.type' in layer_obj['mdns'][key][rr_key]:
            if '16' == layer_obj['mdns'][key][rr_key]['dns.resp.type']:
                txt_list = []
                for rr_val_key in layer_obj['mdns'][key][rr_key].keys():
                    txt_list.append(rr_val_key)
                txt_list.sort()
                for txt_key in txt_list:
                    if ('dns.txt' in txt_key) and ('dns.txt.length' not in txt_key):
                        #print(layer_obj['mdns'][key][rr_key][txt_key])
                        val = val + seperator + layer_obj['mdns'][key][rr_key][txt_key]
                        if 'model=' in layer_obj['mdns'][key][rr_key][txt_key]:
                            model = layer_obj['mdns'][key][rr_key][txt_key].split('=')[1]
    if model:
        val = val + '&&' + model
    return val

def add_list_val(json_obj, key, val):
    if key in json_obj:
        key_str = json_obj[key]
        ptn1 = "_" + val + "_"
        ptn2 = "_" + val + "$"
        ptn3 = "^" + val + "_"
        ptn4 = "^" + val + "$"
        if (not re.search(ptn1, key_str)) and (not re.search(ptn2, key_str)) and (not re.search(ptn3, key_str)) and (not re.search(ptn4, key_str)):
            json_obj[key] = key_str + '_nextpacket_' + val
    else:
        json_obj[key] = val

#for dhcpv6
def req_list2comma_list(raw_list):
    new_str = ''
    nums = raw_list.split(':')
    for i in range(int(len(nums)/2)):
        temp = nums[i*2] + nums[i*2+1]
        temp_num = int(temp, 16)
        if not new_str:
            new_str = str(temp_num)
        else:
            new_str = new_str + ',' + str(temp_num)
    return new_str

#for dhcp
def dhcp_req_list2comma_list(raw_list):
    new_str = ''
    nums = raw_list.split(':')
    for i in range(int(len(nums))):
        temp = nums[i]
        temp_num = int(temp, 16)
        if not new_str:
            new_str = str(temp_num)
        else:
            new_str = new_str + ',' + str(temp_num)
    return new_str

#for udp
def hex_str2char_str(raw_list):
    new_str = ''
    nums = raw_list.split(':')
    for i in range(int(len(nums))):
        temp = nums[i]
        temp_num = int(temp, 16)
        temp_chr = chr(temp_num)
        if not new_str:
            new_str = temp_chr
        else:
            new_str = new_str + temp_chr
    return new_str

def extract_mdns_dot1(raw_str):
    new_str = ''
    packet_list = raw_str.split('_nextpacket_')
    for pkt in packet_list:
        rr_list = pkt.split('_nextrr_')
        for rr in rr_list:
            if not new_str:
                new_str = rr.split(seperator)[0]
            elif (rr.split(seperator)[0] not in new_str):
                new_str = new_str + ',' + rr.split(seperator)[0]
    return new_str

def mac_gene_feature(json_mac_obj, resolved_mac, model):
    if 'feature_label' not in json_mac_obj:
        json_mac_obj['feature_label'] = {}
    json_feature = json_mac_obj['feature_label']
    json_feature['vendor'] = resolved_mac.split('_')[0]
    if 'dhcp' in json_mac_obj:
        if 'request' in json_mac_obj['dhcp']:
            if '12' in json_mac_obj['dhcp']['request']:
                json_feature['dhcp_hostname'] = json_mac_obj['dhcp']['request']['12']
            if '60' in json_mac_obj['dhcp']['request']:
                json_feature['dhcp_vendorclass'] = json_mac_obj['dhcp']['request']['60']
            if '77' in json_mac_obj['dhcp']['request']:
                mod_name = re.findall(r".*Mod=(.+?)\n.*", json_mac_obj['dhcp']['request']['77'])
                if mod_name:
                    json_feature['dhcp_userclass'] = mod_name[0]
        if 'discover' in json_mac_obj['dhcp']:
            if '12' in json_mac_obj['dhcp']['discover']:
                json_feature['dhcp_hostname'] = json_mac_obj['dhcp']['discover']['12']
            if '60' in json_mac_obj['dhcp']['discover']:
                json_feature['dhcp_vendorclass'] = json_mac_obj['dhcp']['discover']['60']
            if '77' in json_mac_obj['dhcp']['discover']:
                mod_name = re.findall(r".*Mod=(.+?)\n.*", json_mac_obj['dhcp']['discover']['77'])
                if mod_name:
                    json_feature['dhcp_userclass'] = mod_name[0]
        if ('ack' in json_mac_obj['dhcp']) or ('offer' in json_mac_obj['dhcp']):
            json_feature['dhcp_router'] = 'router'
    if 'dhcpv6' in json_mac_obj:
        if '39' in json_mac_obj['dhcpv6']:
            json_feature['dhcpv6'] = json_mac_obj['dhcpv6']['39']
    if 'mdns' in json_mac_obj:
        if 'response' in json_mac_obj['mdns']:
            if 'answer' in json_mac_obj['mdns']['response']:
                raw_q_str = json_mac_obj['mdns']['response']['answer'] #query:query, response:answer
                q_name = extract_mdns_dot1(raw_q_str)
                json_feature['mdns_name'] = q_name
        if model:
            json_feature['mdns_model'] = model
    if 'llmnr' in json_mac_obj:
        json_feature['llmnr'] = re.sub(r'_nextpacket_|_nextrr_', ',', json_mac_obj['llmnr'])
    if 'browser' in json_mac_obj:
        json_feature['browser'] = re.sub(r'_nextpacket_|_nextrr_', ',', json_mac_obj['browser'])
    if 'nbns' in json_mac_obj:
        json_feature['nbns'] = re.sub(r'_nextpacket_|_nextrr_', ',', json_mac_obj['nbns'])
    if 'ssdp' in json_mac_obj:
        if 'search' in json_mac_obj['ssdp']:
            if 'user-agent' in json_mac_obj['ssdp']['search']:
                json_feature['ssdp'] = {}
                json_feature['ssdp']['user-agent'] = re.sub(r'_nextpacket_|_nextrr_', ',', json_mac_obj['ssdp']['search']['user-agent'])
            #used for sony tv
            if 'X-AV-Client-Info' in json_mac_obj['ssdp']['search']:
                cn = re.findall(r".*cn=\"(.+?)\";.*", json_mac_obj['ssdp']['search']['X-AV-Client-Info'])[0]
                mn = re.findall(r".*mn=\"(.+?)\";.*", json_mac_obj['ssdp']['search']['X-AV-Client-Info'])[0]
                if cn:
                    if 'ssdp' not in json_feature:
                        json_feature['ssdp'] = {}
                        json_feature['ssdp']['clientcn'] = cn
                    else:
                        json_ssdp_feature = json_feature['ssdp']
                        json_ssdp_feature['clientcn'] = cn
                if mn:
                    if 'ssdp' not in json_feature:
                        json_feature['ssdp'] = {}
                        json_feature['ssdp']['clientmn'] = mn
                    else:
                        json_ssdp_feature = json_feature['ssdp']
                        json_ssdp_feature['clientmn'] = mn
        if 'notify' in json_mac_obj['ssdp']:
            if 'server' in json_mac_obj['ssdp']['notify']:
                if 'ssdp' not in json_feature:
                    json_feature['ssdp'] = {}
                    json_feature['ssdp']['server'] = re.sub(r'_nextpacket_|_nextrr_', ',', json_mac_obj['ssdp']['notify']['server'])
                else:
                    json_ssdp_feature = json_feature['ssdp']
                    json_ssdp_feature['server'] = re.sub(r'_nextpacket_|_nextrr_', ',', json_mac_obj['ssdp']['notify']['server'])
            #used for sony tv
            if 'X-AV-Server-Info' in json_mac_obj['ssdp']['notify']:
                cn = re.findall(r".*cn=\"(.+?)\";.*", json_mac_obj['ssdp']['notify']['X-AV-Server-Info'])[0]
                mn = re.findall(r".*mn=\"(.+?)\";.*", json_mac_obj['ssdp']['notify']['X-AV-Server-Info'])[0]
                if cn:
                    if 'ssdp' not in json_feature:
                        json_feature['ssdp'] = {}
                        json_feature['ssdp']['servercn'] = cn
                    else:
                        json_ssdp_feature = json_feature['ssdp']
                        json_ssdp_feature['servercn'] = cn
                if mn:
                    if 'ssdp' not in json_feature:
                        json_feature['ssdp'] = {}
                        json_feature['ssdp']['servermn'] = mn
                    else:
                        json_ssdp_feature = json_feature['ssdp']
                        json_ssdp_feature['servermn'] = mn
    if 'udp' in json_mac_obj:
        json_feature['udp'] = json_mac_obj['udp']

def mac_gene_priority_feature(json_root_arr, json_feature_arr):
    for mac_obj in json_root_arr:
        mac_feature_obj = find_mac_obj(json_feature_arr, mac_obj['mac'])
        #mac prefix
        mac_addr_obj['vendor'] = mac_obj['feature_label']['vendor']
        #dhcp
        if 'dhcp_hostname' in mac_obj['feature_label']:
            dhcp_hostname_obj['dhcp_hostname'] = mac_obj['feature_label']['dhcp_hostname']
        if 'dhcp_userclass' in mac_obj['feature_label']:
            dhcp_userclass_obj['dhcp_userclass'] = mac_obj['feature_label']['dhcp_userclass']
        if 'dhcp_vendorclass' in mac_obj['feature_label']:
            dhcp_vendorclass_obj['dhcp_vendorclass'] = mac_obj['feature_label']['dhcp_vendorclass']
        #dhcpv6
        if 'dhcpv6' in mac_obj['feature_label']:
            dhcpv6_obj['dhcpv6'] = mac_obj['feature_label']['dhcpv6']
        #mdns
        if 'mdns_name' in mac_obj['feature_label']:
            mdns_name_obj['mdns_name'] = mac_obj['feature_label']['mdns_name']
        if 'mdns_model' in mac_obj['feature_label']:
            mdns_model_obj['mdns_model'] = mac_obj['feature_label']['mdns_model']
        #ssdp

#read csv to dict
def csv2dict(in_file,key,value):
    new_dict = {}
    with open(in_file, 'r') as f:
        reader = csv.reader(f, delimiter=',')
        fieldnames = next(reader)
        reader = csv.DictReader(f, fieldnames=fieldnames, delimiter=',')
        for row in reader:
            new_dict[row[key]] = row[value]
    return new_dict

for root, dirs, files in os.walk(pkt_dir, False):
    if files:
        for line in files:
            if '.json' not in line:
                #file_cnt = file_cnt + 1
                file_cnt = line.split('-')[0]
		        #if (int(file_cnt) <= 34):
		        #    continue
                if (int(file_cnt) <= 666):
                    continue
                filepath = os.path.join(root, line)
                file_name = filepath

                csv_file = './support/knowledge.csv'
                output_json_name = 'train_data' + str(file_cnt) + '.json'
                output_csv_name = 'result' + str(file_cnt) +'.csv'

                #create json_root_arr
                json_root_arr = []
                json_feature_arr = []

                bootp_option_type_cnt = 0
                bootp_option_type_tree_cnt = 0
                bootp_option_request_list_item_cnt = 0
                http_request_line_cnt = 0
                dhcpv6_option_type_cnt = 0
                dns_txt_cnt = 0
                http_cnt = 0

                #remove key duplicate
                wf = open(file_name+'.json', 'w+', encoding='utf-8')
                with open(file_name,'r') as raw_f:
                    for line in raw_f.readlines():
                        if 'ip6.arpa' in line:
                            line = re.sub(r'(?<=[\"\s])[0-9A-F\.]{64}(?=ip6\.arpa)', 'ipv6,', line)
                        line = re.sub(r'((25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(25[0-5]|2[0-4]\d|[01]?\d\d?)', 'ipv4', line)
                        if ('dhcp' not in line) and ('bootp' not in line) and ('data.data' not in line):
                            line = re.sub(r'(?<![:])([a-f0-9]{1,4}(:[a-f0-9]{1,4}){7}|[a-f0-9]{1,4}(:[a-f0-9]{1,4}){0,7}::[a-f0-9]{0,4}(:[a-f0-9]{1,4}){0,7})(?!:)', 'ipv6', line)
                        if 'bootp.option.type\"' in line:
                            bootp_option_type_cnt += 1
                            new_str = str(bootp_option_type_cnt) + ',bootp.option.type'
                            new_line = line.replace('bootp.option.type', new_str)
                        elif 'bootp.option.type_tree' in line:
                            bootp_option_type_tree_cnt += 1
                            new_str = str(bootp_option_type_tree_cnt) + ',bootp.option.type_tree'
                            new_line = line.replace('bootp.option.type_tree', new_str)
                        elif 'bootp.option.request_list_item' in line:
                            bootp_option_request_list_item_cnt += 1
                            new_str = 'bootp.option.request_list_item,' + str(bootp_option_request_list_item_cnt)
                            new_line = line.replace('bootp.option.request_list_item', new_str)
                        #ssdp
                        elif 'http.request.line' in line:
                            http_request_line_cnt += 1
                            new_str = 'http.request.line' + str(http_request_line_cnt)
                            new_line = line.replace('http.request.line', new_str)
                        elif 'dhcpv6.option.type' in line:
                            dhcpv6_option_type_cnt += 1
                            new_str = str(dhcpv6_option_type_cnt) + ',dhcpv6.option.type'
                            new_line = line.replace('dhcpv6.option.type', new_str)
                        elif 'dns.txt' in line:
                            dns_txt_cnt += 1
                            new_str = str(dns_txt_cnt) + ',dns.txt'
                            new_line = line.replace('dns.txt', new_str)
                        #mdns
                        elif ('http' in line) and ('class' not in line):
                            http_cnt += 1
                            new_str = str(http_cnt) + ',http'
                            new_line = re.sub(r'http(?!:)', new_str, line)
                        else:
                            new_line = line
                        wf.write(new_line)
                wf.close()

                with open(file_name+'.json', 'r') as load_f:
                    pkt_arr = json.load(load_f)

                for pkt_obj in pkt_arr:
                    model = ''

                    src_obj = pkt_obj['_source']
                    layer_obj = src_obj['layers']

                    #src mac:
                    eth_obj = layer_obj['eth']
                    mac = eth_obj['eth.src']
                    json_mac_obj = find_mac_obj(json_root_arr, mac)

                    #resolved src mac:
                    mac_tree_obj = eth_obj['eth.src_tree']
                    resolved_mac = mac_tree_obj['eth.src_resolved']

                    #protocol:
                    ptcl = extract_ptcl(layer_obj)
                    gene_ptcl_obj(json_mac_obj, ptcl)

                    if('bootp' == ptcl):
                        bootp_keys = layer_obj['bootp'].keys()
                        add_ptcl_obj(json_mac_obj, 'dhcp')
                        json_dhcp_obj = json_mac_obj['dhcp']
                        json_dhcp_type_obj = {}

                        flag_cnt = 0
                        while(flag_cnt < bootp_option_type_cnt):
                            flag_cnt += 1
                            option_key = str(flag_cnt) + ',' + 'bootp.option.type'
                            val_key = str(flag_cnt) + ',' + 'bootp.option.type_tree'
                            add_key = ''
                            add_val = ''
                            for key in bootp_keys:
                                if(option_key == key):
                                    add_key = layer_obj['bootp'][key]
                            if('53' == add_key):
                                for key in bootp_keys:
                                    if(val_key == key):
                                        add_val = layer_obj['bootp'][key]['bootp.option.dhcp']

                            #add to json
                            if('53' == add_key) and ('1' == add_val):
                                dhcp_type = 'discover'
                                add_ptcl_obj(json_dhcp_obj, dhcp_type)
                                json_dhcp_type_obj = json_dhcp_obj[dhcp_type]
                            elif('53' == add_key) and ('3' == add_val):
                                dhcp_type = 'request'
                                add_ptcl_obj(json_dhcp_obj, dhcp_type)
                                json_dhcp_type_obj = json_dhcp_obj[dhcp_type]
                            elif('53' == add_key) and ('6' == add_val):
                                dhcp_type = 'nak'
                                add_ptcl_obj(json_dhcp_obj, dhcp_type)
                                json_dhcp_type_obj = json_dhcp_obj[dhcp_type]
                            elif('53' == add_key) and ('2' == add_val):
                                dhcp_type = 'offer'
                                add_ptcl_obj(json_dhcp_obj, dhcp_type)
                                json_dhcp_type_obj = json_dhcp_obj[dhcp_type]
                            elif('53' == add_key) and ('5' == add_val):
                                dhcp_type = 'ack'
                                add_ptcl_obj(json_dhcp_obj, dhcp_type)
                                json_dhcp_type_obj = json_dhcp_obj[dhcp_type]
                            elif('53' == add_key) and ('8' == add_val):
                                dhcp_type = 'inform'
                                add_ptcl_obj(json_dhcp_obj, dhcp_type)
                                json_dhcp_type_obj = json_dhcp_obj[dhcp_type]

                        option_cnt = 0
                        series_str = ''
                        while(option_cnt < bootp_option_type_cnt):
                            option_cnt += 1
                            option_key = str(option_cnt) + ',' + 'bootp.option.type'
                            val_key = str(option_cnt) + ',' + 'bootp.option.type_tree'
                            add_key = ''
                            add_val = ''
                            for key in bootp_keys:
                                if (option_key == key):
                                    add_key = layer_obj['bootp'][key]
                                    if not series_str:
                                        series_str = add_key
                                    else:
                                        series_str = series_str + ',' + add_key
                            if('55' == add_key):
                                for key in bootp_keys:
                                    if(val_key == key):
                                        add_val = layer_obj['bootp'][key]['bootp.option.value']
                            elif('81' == add_key):
                                for key in bootp_keys:
                                    if(val_key == key) and ('bootp.fqdn.name' in layer_obj['bootp'][key]):
                                        add_val = layer_obj['bootp'][key]['bootp.fqdn.name']
                            elif('77' == add_key):
                                for key in bootp_keys:
                                    if(val_key == key):
                                        raw_str = layer_obj['bootp'][key]['bootp.option.user_class_tree']['bootp.option.user_class.data']
                                        add_val = hex_str2char_str(raw_str)
                            elif('0' != add_key):
                                for key in bootp_keys:
                                    if (val_key == key) and (len(layer_obj['bootp'][key]) > 1):
                                        tree_keys = layer_obj['bootp'][key].keys()
                                        for tree_key in tree_keys:
                                            if('bootp.option.length' != tree_key) and ('bootp.option.value' != tree_key) and ('bootp.hw.type' != tree_key):
                                                add_val = layer_obj['bootp'][key][tree_key]
                                                if mac == add_val:
                                                    add_val = resolved_mac.split('_')[0]

                            if('55' == add_key):
                                add_val_str = dhcp_req_list2comma_list(add_val)
                                add_list_val(json_dhcp_type_obj, add_key, add_val_str)
                            elif('0' != add_key) and ('' != add_key):
                                add_ptcl_val(json_dhcp_type_obj, add_key, add_val.replace('.', ','))
                        add_ptcl_val(json_dhcp_type_obj, 'series', series_str)

                    elif('dhcpv6' == ptcl):
                        dhcpv6_keys = layer_obj['dhcpv6'].keys()
                        add_ptcl_obj(json_mac_obj, ptcl)
                        json_dhcpv6_obj = json_mac_obj[ptcl]
                        series_str = ''
                        opt_cnt = 0
                        while(opt_cnt < dhcpv6_option_type_cnt):
                            opt_cnt += 1
                            add_key = ''
                            add_val = ''
                            type_key = str(opt_cnt) + ',' + 'dhcpv6.option.type'
                            for key in dhcpv6_keys:
                                if ('dhcpv6.msgtype' != key and 'dhcpv6.xid' != key):
                                    type_trees = layer_obj['dhcpv6'][key].keys()
                                    for type_tree in type_trees:
                                        if (type_key == type_tree):
                                            add_key = layer_obj['dhcpv6'][key][type_tree]
                                            if not series_str:
                                                series_str = add_key
                                            else:
                                                series_str = series_str + ',' + add_key

                                            if('39' == add_key):
                                                add_ptcl_val(json_dhcpv6_obj, add_key, layer_obj['dhcpv6'][key]['dhcpv6.client_fqdn'].replace('.', ','))
                                            elif('6' == add_key):
                                                req_list = layer_obj['dhcpv6'][key]['dhcpv6.option.value']
                                                req_str = req_list2comma_list(req_list)
                                                add_list_val(json_dhcpv6_obj, add_key, req_str)
                                            else:
                                                add_ptcl_val(json_dhcpv6_obj, add_key, 'null')
                        add_ptcl_val(json_dhcpv6_obj, 'series', series_str)

                    if('mdns' == ptcl):
                        mdns_keys = layer_obj['mdns'].keys()
                        add_ptcl_obj(json_mac_obj, ptcl)
                        json_mdns_obj = json_mac_obj[ptcl]
                        json_mdns_type_obj = {}
                        for key in mdns_keys:
                            if('dns.flags' == key):
                                if('0x00000000' == layer_obj['mdns'][key]):
                                    mdns_type = 'query'
                                    add_ptcl_obj(json_mdns_obj, mdns_type)
                                    json_mdns_type_obj = json_mdns_obj[mdns_type]
                                elif('0x00008400' == layer_obj['mdns'][key]):
                                    mdns_type = 'response'
                                    add_ptcl_obj(json_mdns_obj, mdns_type)
                                    json_mdns_type_obj = json_mdns_obj[mdns_type]
                        for key in mdns_keys:
                            if('Queries' == key):
                                q_val = ''
                                query_val = add_mdns_rrs(layer_obj, key)
                                if '&&' in query_val:
                                    q_val = add_mdns_rrs(layer_obj, key).split('&&')[0]
                                    model = add_mdns_rrs(layer_obj, key).split('&&')[1]
                                else:
                                    q_val = query_val
                                add_ptcl_val(json_mdns_type_obj, 'query', q_val)
                            elif('Answers' == key):
                                a_val = ''
                                answer_val = add_mdns_rrs(layer_obj, key)
                                if '&&' in answer_val:
                                    a_val = add_mdns_rrs(layer_obj, key).split('&&')[0]
                                    model = add_mdns_rrs(layer_obj, key).split('&&')[1]
                                else:
                                    a_val = answer_val
                                add_ptcl_val(json_mdns_type_obj, 'answer', a_val)
                            elif('Additional records' == key):
                                add_val = ''
                                addition_val = add_mdns_rrs(layer_obj, key)
                                if '&&' in addition_val:
                                    add_val = add_mdns_rrs(layer_obj, key).split('&&')[0]
                                    model = add_mdns_rrs(layer_obj, key).split('&&')[1]
                                else:
                                    add_val = addition_val
                                add_ptcl_val(json_mdns_type_obj, 'additional', add_val)
                            elif('Authoritative nameservers' == key):
                                auth_val = ''
                                authentity_val = add_mdns_rrs(layer_obj, key)
                                if '&&' in authentity_val:
                                    auth_val = add_mdns_rrs(layer_obj, key).split('&&')[0]
                                    model = add_mdns_rrs(layer_obj, key).split('&&')[1]
                                else:
                                    auth_val = authentity_val
                                add_ptcl_val(json_mdns_type_obj, 'authoritative', auth_val)
                    elif('llmnr' == ptcl):
                        if isinstance(layer_obj['llmnr']['Queries'], str):
                            print(mac)
                        else:
                            llmnr_keys = layer_obj['llmnr']['Queries'].keys()
                            for key in llmnr_keys: #only one key
                                llmnr_qname = key.split(':')[0]
                            add_ptcl_val(json_mac_obj, ptcl, llmnr_qname)
                    elif('browser' == ptcl):
                        smb_srcname = layer_obj['nbdgm']['nbdgm.source_name']  #only one sourcename
                        add_ptcl_val(json_mac_obj, ptcl, smb_srcname)
                    elif('nbns' == ptcl):
                        if 'Queries' in layer_obj['nbns']:
                            if isinstance(layer_obj['nbns']['Queries'], str):
                                print(layer_obj['nbns']['Queries'])
                            else:
                                nbns_keys = layer_obj['nbns']['Queries'].keys()
                                for key in nbns_keys:
                                    nbns_qname = key.split(':')[0]
                                add_ptcl_val(json_mac_obj, ptcl, nbns_qname)
                    elif('ssdp' == ptcl):
                        ssdp_keys = layer_obj['ssdp'].keys() #key is m-search or notify
                        add_ptcl_obj(json_mac_obj, ptcl)
                        json_ssdp_obj = json_mac_obj[ptcl]
                        json_ssdp_type_obj = {}
                        request_line_list = []
                        ssdp_type = ''
                        for key in ssdp_keys:
                            if('M-SEARCH' == key.split(' ')[0]):
                                ssdp_type = 'search'
                                add_ptcl_obj(json_ssdp_obj, ssdp_type)
                                json_ssdp_type_obj = json_ssdp_obj[ssdp_type]
                            elif('NOTIFY' == key.split(' ')[0]):
                                ssdp_type = 'notify'
                                add_ptcl_obj(json_ssdp_obj, ssdp_type)
                                json_ssdp_type_obj = json_ssdp_obj[ssdp_type]
                        for key in ssdp_keys:
                            if ('search' == ssdp_type) and ('http.request.line' in key):
                                request_line_list.append(key)
                            elif ('notify' == ssdp_type) and ('http' in key):
                                request_line_list.append(key)
                        request_line_list.sort()
                        series_str = ''
                        add_key = ''
                        add_val = ''
                        for item in request_line_list:
                            if 'notify' == ssdp_type:
                                if 'http.host' in item:
                                    add_key = 'host'
                                    add_val = layer_obj['ssdp'][item]
                                elif 'http.cache_control' in item:
                                    add_key = 'cache_control'
                                    add_val = layer_obj['ssdp'][item]
                                elif 'http.location' in item:
                                    add_key = 'location'
                                    add_val = layer_obj['ssdp'][item]
                                elif 'http.server' in item:
                                    add_key = 'server'
                                    add_val = layer_obj['ssdp'][item]
                                elif 'http.request.full_uri' in item:
                                    add_key = 'full_uri'
                                    add_val = layer_obj['ssdp'][item]
                                elif 'http.unknown_header' in item:
                                    add_key = layer_obj['ssdp'][item].split(':')[0]
                                    if len(layer_obj['ssdp'][item].split(':')) > 1:
                                        add_val = layer_obj['ssdp'][item].split(':')[1]
                                    else:
                                        add_val = 'nak'
                                else:
                                    add_key = ''
                            elif 'search' == ssdp_type:
                                add_key = layer_obj['ssdp'][item].split(':')[0]
                                add_val = layer_obj['ssdp'][item].split(':')[1]

                            if add_key:
                                if (not series_str):
                                    series_str = add_key
                                else:
                                    series_str = series_str + ',' + add_key

                            if('user-agent' == add_key):
                                add_val = add_val.replace(' ', ',')
                            if add_key:
                                add_ptcl_val(json_ssdp_type_obj, add_key, add_val.replace(':', ','))
                        add_ptcl_val(json_ssdp_type_obj, 'series', series_str)
                    elif 'udp' in ptcl:
                        if 'data' in layer_obj:
                            raw_data_str = layer_obj['data']['data.data']
                            data_str = hex_str2char_str(raw_data_str)
                            device_name = re.findall(r".*DEVICE_NAME=(.+?)\n.*", data_str)
                            if device_name:
                                add_ptcl_val(json_mac_obj, 'udp', device_name[0])
                    mac_gene_feature(json_mac_obj, resolved_mac, model)
                #generate priority feature
                #mac_gene_priority_feature(json_root_arr, json_feature_arr)

                with open(output_json_name, 'w') as outfile:
                    json.dump(json_root_arr, outfile, indent=4)

                #read in knowledge csv
                vendor_know_dict = csv2dict(csv_file, 'vendor_key', 'vendor')
                type_know_dict = csv2dict(csv_file, 'type_key', 'type')
                model_know_dict = csv2dict(csv_file, 'model_key', 'model')

                #add prefix
                vendor_know_arr = []
                type_know_arr = []
                model_know_arr = []
                for vendor_key in vendor_know_dict.keys():
                    vendor_know_arr.append(vendor_key)
                    vendor_know_arr.sort(reverse = True)
                for type_key in type_know_dict.keys():
                    type_know_arr.append(type_key)
                    type_know_arr.sort(reverse = True)
                for model_key in model_know_dict.keys():
                    model_know_arr.append(model_key)
                    model_know_arr.sort(reverse = True)

                #generate csv file
                #mac|feature
                mac_list = []
                feature_list = []
                mac_prefix_list = []
                vendor_list = []
                type_list = []
                model_list = []
                for mac_obj in json_root_arr:
                    mac_list.append(mac_obj['mac'])

                    #mac_prefix
                    mac_arr = mac_obj['mac'].split(':')
                    mac_prefix = mac_arr[0]
                    mac_prefix = mac_prefix + mac_arr[1]
                    mac_prefix = mac_prefix + mac_arr[2]

                    feature_str = mac_prefix + ','
                    for key in mac_obj['feature_label'].keys():
                        if 'ssdp' == key:
                            for type_key in mac_obj['feature_label']['ssdp'].keys():
                                if not feature_str:
                                    feature_str = mac_obj['feature_label']['ssdp'][type_key].replace('\r\n', '')
                                else:
                                    feature_str = feature_str + ',' + mac_obj['feature_label']['ssdp'][type_key].replace('\r\n', '')
                        else:
                            if not feature_str:
                                feature_str = mac_obj['feature_label'][key].replace('\r\n', '')
                            else:
                                feature_str = feature_str + ',' + mac_obj['feature_label'][key].replace('\r\n', '')
                    feature_list.append(feature_str)

                    #mac_prefix label:
                    mac_prefix_list.append(mac_prefix)

                    #vendor label:
                    vendor = ''
                    for vendor_key in vendor_know_arr:
                        if vendor_key.lower() in feature_str.lower():
                            vendor = vendor_know_dict[vendor_key]
                            break
                    vendor_list.append(vendor)
                    #type label:
                    dev_type = ''
                    for type_key in type_know_arr:
                        if type_key.lower() in feature_str.lower():
                            dev_type = type_know_dict[type_key]
                            break
                        if 'router' in feature_str.lower():
                            #dev_type = vendor + '_router'
                            print(str(file_cnt) + ' : router')
                            break
                    type_list.append(dev_type)
                    #model label:
                    model = ''
                    for model_key in model_know_arr:
                        if model_key.lower() in feature_str.lower():
                            model = model_know_dict[model_key]
                            break
                    model_list.append(model)

                dataframe = pd.DataFrame({'mac':mac_list, 'feature':feature_list, 'mac_prefix':mac_prefix_list, 'vendor':vendor_list, 'type':type_list, 'model':model_list}, columns=["mac", "feature", "mac_prefix", "vendor", "type", "model"])
                dataframe.to_csv(output_csv_name, sep=',')
                #print(json_root_arr)


# coding: utf-8
__author__ = 'Stiller'
from django.conf.urls import patterns, url
from commons import views

routers = patterns(
    '',
    url(r'^mis/unsupport$', views.unsupport, name='Unsupport'),
    url(r'^mis/survey$', views.view_survey),
    url(r'^mis/post_survey$', views.post_survey),
)


from heapq import heappush, heappop
from random import randrange
Q = []
for i in range(10):
	heappush(Q, randrange(100))

def sift_up(heap, startpos, pos):
	newitem = heap[pos]
	while pos > startpos:
		parentpos = (pos - 1) >> 1
		parent = heap[parentpos]
		if parent <= newitem:
			break
		heap[pos] = parent
		pos = parentpos
	heap[pos] = newitem

from heapq import heapify, heappush, heappop
from itertools import count
def huffman(seq, frq):
	num = count()
	trees = list(zip(frq, num, seq))
	heapify(trees)
	while len(trees) > 1:
		fa, _, a = heappop(trees)
		fb, _, b = heappop(trees)
		n = next(num)
		heappush(trees, (fa+fb, n, [a, b]))
	return trees[0][-1]
# num ensures valid ordering # A min-heap based on freq
# Until all are combined
# Get the two smallest trees
# Combine and re-add them


# -*- coding: utf-8 -*-
"""
Created on Thu Mar 30 12:26:41 2017

@author: mqbpwbe2
"""

from multiprocessing.dummy import Pool
from pathlib import Path
from updash_lib.read_spectrum import read_spectrum
import numpy as np
import matplotlib.pyplot as plt
import datetime

fpath = 'Spectra/2017-09-06/spectro_gui_out(1)/USB2+H15972/spectra/spectrum_'

start = 0
stop = 243
global ind
ind = 300
global val
val = np.ndarray(2)

def get_val(i):

    n = '{num:05d}'.format(num=i)
    # Construct full filepath
    fname = fpath + n + '.txt'

    # Check file exists, if true then read
    if Path(fname).is_file():
        global grid
        grid, y, data_date, data_time = read_spectrum(fname)

        return y[ind]

def get_times(i):

    n = '{num:05d}'.format(num=i)
    # Construct full filepath
    fname = fpath + n + '.txt'

    # Check file exists, if true then read
    if Path(fname).is_file():
        global grid
        grid, y, data_date, data_time = read_spectrum(fname)

        time = datetime.datetime.strptime(data_time,'%H:%M:%S')

        return time

pool = Pool()

vals = pool.map(get_val,range(start,stop+1))

# Create figure for the plot
plt.rcParams.update({'font.size': 18})
fig, ax = plt.subplots(1)
ax.plot(range(start,stop+1),vals)
ax.set_xlabel('Spectrum number')
ax.set_ylabel('Value at ' + str(grid[ind]) + ' nm')
plt.show()


#!/usr/bin/env python3

import argparse

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('FILE', nargs = '?', default = 'input')
    args = parser.parse_args()
    with open(args.FILE, 'r') as f:
        a = int(f.readline().split(' ')[-1])
        b = int(f.readline().split(' ')[-1])
    f_a = 16807
    f_b = 48271

    def gen(i, f):
        while True:
            i = (i*f)%2147483647
            yield i
    gen_a = gen(a, f_a)
    gen_b = gen(b, f_b)

    N = 40000000
    matches = 0
    for _ in range(N):
        a_bin = next(gen_a)
        b_bin = next(gen_b)
        matches += (a_bin%65536 == b_bin%65536)
    print(matches)

    def gen2(i, f, n):
        while True:
            i = (i*f)%2147483647
            if not i%n:
                yield i

    gen_a = gen2(a, f_a, 4)
    gen_b = gen2(b, f_b, 8)

    N = 5000000
    matches = 0
    for _ in range(N):
        a_bin = next(gen_a)
        b_bin = next(gen_b)
        matches += (a_bin%65536 == b_bin%65536)
    print(matches)


from time import sleep
from django.db import connections
from django.db.utils import OperationalError
from django.core.management.base import BaseCommand, CommandError

class Command(BaseCommand):
    help = 'Closes the specified poll for voting'

    def handle(self, *args, **options):
        conn = connections['default']
        connected = False
        while not connected:
            try:
                c = conn.cursor()
            except OperationalError:
                sleep(2)
                print("Waiting for the database")
            else:
                print("Connected to the database")
                connected = True


# Copyright (c) 2014 CodiLime
__author__ = 'Piotr Niedzwiedz <piotr@codilime.com>'

import Image
import argparse
import logging
import os

import ch24.common.helpers

LOG = logging.getLogger()

def _common_img_format(filename):
    COMMON_IMG_FORMATS = ['.jpg', '.jpeg', '.png']
    _, extension = os.path.splitext(filename)
    extension = extension.lower()
    return extension in COMMON_IMG_FORMATS

def image_from_imf(lines):
    """Creates Matrix Image from IMF
    """
    RGBA_OPAQUE = 255
    height, width, color_model = lines[0].split()
    height = eval(height)
    width = eval(width)
    color_model = color_model.lower()
    assert(color_model in ['rgb', 'rgba', 'grayscale'])
    img = Image.new("RGBA", (width, height), "white")
    pixels = img.load()
    for y in xrange(height):
        line_pixels = lines[y+1].split()
        for x in xrange(width):
            if color_model == 'rgb':
                pixel = line_pixels[:3]
                line_pixels = line_pixels[3:]
                pixel.append(RGBA_OPAQUE)
            elif color_model == 'rgba':
                pixel = line_pixels[:4]
                line_pixels = line_pixels[4:]
            else:  # grayscale
                pixel = line_pixels[:1]
                line_pixels = line_pixels[1:]
                pixel = pixel[0], pixel[0], pixel[0], RGBA_OPAQUE
            assert len(pixel) == 4
            pixels[(x, y)] = tuple(map(int, pixel))
    return img

def load_img_from_file(filename):
    if _common_img_format(filename):
        LOG.debug('Loading from file in common image format.')
        image = Image.open(filename)
        image = image.convert('RGBA')
    else:
        LOG.debug('Loading from file in Integer matrix format.')
        fd = open(filename, 'r')
        image = image_from_imf(fd.readlines())
    return image

def img_to_imf(image, color_model):
    width, height = image.size
    lines = []
    lines.append('{} {} {}\n'.format(height, width, color_model))
    pixels = image.load()
    for y in xrange(height):
        line = []
        for x in xrange(width):
            pixel = pixels[(x,y)]
            if isinstance(pixel, tuple):
                line.append(' '.join(map(str, pixel)))
            else:
                line.append(str(pixel))
        lines.append(' '.join(line) + '\n')
    return lines

def save_to_file(filename, image, color_model):
    if color_model == 'grayscale':
        image = image.convert('L')
    elif color_model == 'rgb':
        image = image.convert('RGB')
    else:
        assert color_model == 'rgba'
    if _common_img_format(filename):
        LOG.debug('Saving to file in common image format.')
        image.save(filename)
    else:
        LOG.debug('Loading from file in Integer matrix format.')
        try:
            fd = open(filename, 'w')
            fd.writelines(img_to_imf(image, color_model))
        finally:
            fd.close()

def main():
    parser = argparse.ArgumentParser(description='Image converter.')
    parser.add_argument('input_file', metavar='input_file', type=str, help='path to the input file')
    parser.add_argument('output_file', metavar='output_file', type=str, help='path to the output file')
    parser.add_argument('color_model',
                        metavar='color_model',
                        type=str,
                        help='has to have one of the following values: "grayscale", "rgb" and "rgba".')
    parser.add_argument('-d',
                        '--debug',
                        dest='debug',
                        action='store_const',
                        const=True,
                        default=False,
                        help='print debugging information')
    args = parser.parse_args()

    ch24.common.helpers.set_up_logging(args.debug)

    image = load_img_from_file(args.input_file)
    save_to_file(args.output_file, image, args.color_model.lower())


import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.tree import DecisionTreeClassifier
# from sklearn.neighbors import KNeighborsClassifier
# Create dataframe contain .csv
url = './data_mobile.csv'
df = pd.read_csv(url, encoding='utf-8')
X = df.iloc[:, 1] # Get all data at column 1; iloc[:,:4] => get from column 1 -> 3
Y = df.iloc[:, 2] # Get all data at column 2

X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2) # test size 20%
cls = DecisionTreeClassifier()
cls.fit(X_train, Y_train) # X_train, Y_train must be number
comment = ['Điện thoại màn hình quá tệ']


import pandas as pd
import numpy as np
import tensorflow as tf
import os
from keras.preprocessing.image import ImageDataGenerator, array_to_img, img_to_array, load_img
import random
from skimage.transform import resize

from keras.models import Model, load_model
from keras.layers import Input,Dropout,BatchNormalization,Activation,Add
from keras.layers.core import Lambda
from keras.layers.convolutional import Conv2D, Conv2DTranspose
from keras.layers.pooling import MaxPooling2D
from keras.layers.merge import concatenate
from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau
from keras import backend as K
#
def mean_iou(y_true, y_pred):
    prec = []
    for t in np.arange(0.5, 1.0, 0.05):
        y_pred_ = tf.to_int32(y_pred > t)
        score, up_opt = tf.metrics.mean_iou(y_true, y_pred_, 2)
        K.get_session().run(tf.local_variables_initializer())
        with tf.control_dependencies([up_opt]):
            score = tf.identity(score)
        prec.append(score)
    return K.mean(K.stack(prec), axis=0)
#
im_width = 128
im_height = 128
im_chan = 1

path_train = '/home/luxuyan2016/SaltMaskCompetition/train/'
train_ids = next(os.walk(path_train+"images"))[2]

# Get and resize train images and masks
X_train = np.zeros((len(train_ids), im_height, im_width, im_chan), dtype=np.uint8)
Y_train = np.zeros((len(train_ids), im_height, im_width, 1), dtype=np.bool)

for n, id_ in enumerate(train_ids):
    path = path_train
    img = load_img(path + '/images/' + id_)
    x = img_to_array(img)[:,:,1]
    x = resize(x, (128, 128, 1), mode='constant', preserve_range=True)
    X_train[n] = x
    mask = img_to_array(load_img(path + '/masks/' + id_))[:,:,1]
    Y_train[n] = resize(mask, (128, 128, 1), mode='constant', preserve_range=True)

ACTIVATION = "relu"

def convolution_block(x, filters, size, strides=(1,1), padding='same', activation=True):
    x = Conv2D(filters, size, strides=strides, padding=padding)(x)
    x = BatchNormalization()(x)
    if activation == True:
        x = Activation(ACTIVATION)(x)
    return x

def residual_block(blockInput, num_filters=16):
    x = Activation(ACTIVATION)(blockInput)
    x = BatchNormalization()(x)
    x = convolution_block(x, num_filters, (3,3))
    x = convolution_block(x, num_filters, (3,3), activation=False)
    x = Add()([x, blockInput])
    return x

# Build model
def build_model(input_layer, start_neurons, DropoutRatio = 0):
    # 128 -> 64
    conv1 = Conv2D(start_neurons * 1, (3, 3), activation=None, padding="same")(input_layer)
    conv1 = residual_block(conv1, start_neurons * 1)
    conv1 = residual_block(conv1, start_neurons * 1)
    conv1 = Activation(ACTIVATION)(conv1)
    pool1 = MaxPooling2D((2, 2))(conv1)
    pool1 = Dropout(DropoutRatio / 2)(pool1)

    # 64 -> 32
    conv2 = Conv2D(start_neurons * 2, (3, 3), activation=None, padding="same")(pool1)
    conv2 = residual_block(conv2, start_neurons * 2)
    conv2 = residual_block(conv2, start_neurons * 2)
    conv2 = Activation(ACTIVATION)(conv2)
    pool2 = MaxPooling2D((2, 2))(conv2)
    pool2 = Dropout(DropoutRatio)(pool2)

    # 32 -> 16
    conv3 = Conv2D(start_neurons * 4, (3, 3), activation=None, padding="same")(pool2)
    conv3 = residual_block(conv3, start_neurons * 4)
    conv3 = residual_block(conv3, start_neurons * 4)
    conv3 = Activation(ACTIVATION)(conv3)
    pool3 = MaxPooling2D((2, 2))(conv3)
    pool3 = Dropout(DropoutRatio)(pool3)

    # 16 -> 8
    conv4 = Conv2D(start_neurons * 8, (3, 3), activation=None, padding="same")(pool3)
    conv4 = residual_block(conv4, start_neurons * 8)
    conv4 = residual_block(conv4, start_neurons * 8)
    conv4 = Activation(ACTIVATION)(conv4)
    pool4 = MaxPooling2D((2, 2))(conv4)
    pool4 = Dropout(DropoutRatio)(pool4)

    # Middle
    convm = Conv2D(start_neurons * 16, (3, 3), activation=None, padding="same")(pool4)
    convm = residual_block(convm, start_neurons * 16)
    convm = residual_block(convm, start_neurons * 16)
    convm = Activation(ACTIVATION)(convm)

    # 8 -> 16
    deconv4 = Conv2DTranspose(start_neurons * 8, (2, 2), strides=(2, 2), padding="same")(convm)
    uconv4 = concatenate([deconv4, conv4])
    uconv4 = Dropout(DropoutRatio)(uconv4)

    uconv4 = Conv2D(start_neurons * 8, (3, 3), activation=None, padding="same")(uconv4)
    uconv4 = residual_block(uconv4, start_neurons * 8)
    uconv4 = residual_block(uconv4, start_neurons * 8)
    uconv4 = Activation(ACTIVATION)(uconv4)

    # 16 -> 32
    deconv3 = Conv2DTranspose(start_neurons * 4, (2, 2), strides=(2, 2), padding="same")(uconv4)
    # deconv3 = Conv2DTranspose(start_neurons * 4, (3, 3), strides=(2, 2), padding="valid")(uconv4)
    uconv3 = concatenate([deconv3, conv3])
    uconv3 = Dropout(DropoutRatio)(uconv3)

    uconv3 = Conv2D(start_neurons * 4, (3, 3), activation=None, padding="same")(uconv3)
    uconv3 = residual_block(uconv3, start_neurons * 4)
    uconv3 = residual_block(uconv3, start_neurons * 4)
    uconv3 = Activation(ACTIVATION)(uconv3)

    # 32 -> 64
    deconv2 = Conv2DTranspose(start_neurons * 2, (2, 2), strides=(2, 2), padding="same")(uconv3)
    uconv2 = concatenate([deconv2, conv2])

    uconv2 = Dropout(DropoutRatio)(uconv2)
    uconv2 = Conv2D(start_neurons * 2, (3, 3), activation=None, padding="same")(uconv2)
    uconv2 = residual_block(uconv2, start_neurons * 2)
    uconv2 = residual_block(uconv2, start_neurons * 2)
    uconv2 = Activation(ACTIVATION)(uconv2)

    # 64 -> 128
    deconv1 = Conv2DTranspose(start_neurons * 1, (2, 2), strides=(2, 2), padding="same")(uconv2)
    # deconv1 = Conv2DTranspose(start_neurons * 1, (3, 3), strides=(2, 2), padding="valid")(uconv2)
    uconv1 = concatenate([deconv1, conv1])

    uconv1 = Dropout(DropoutRatio)(uconv1)
    uconv1 = Conv2D(start_neurons * 1, (3, 3), activation=None, padding="same")(uconv1)
    uconv1 = residual_block(uconv1, start_neurons * 1)
    uconv1 = residual_block(uconv1, start_neurons * 1)
    uconv1 = Activation(ACTIVATION)(uconv1)

    uconv1 = Dropout(DropoutRatio / 2)(uconv1)
    output_layer = Conv2D(1, (1, 1), padding="same", activation="sigmoid")(uconv1)

    return output_layer
#
# model
input_layer = Input((im_height, im_width, im_chan))
s = Lambda(lambda x: x / 255)(input_layer)
output_layer = build_model(s, 16, 0.5)
#
model = Model(input_layer, output_layer)
model.summary()
model.compile(loss="binary_crossentropy", optimizer="adam", metrics=["acc", mean_iou])
early_stopping = EarlyStopping(monitor='val_acc', mode = 'max', patience=20, verbose=1)
model_checkpoint = ModelCheckpoint("./unet_best1.model", monitor='val_acc',
                                   mode = 'max', save_best_only=True, verbose=1)
reduce_lr = ReduceLROnPlateau(monitor='val_acc', mode = 'max',factor=0.2, patience=5, min_lr=0.00001, verbose=1)
# reduce_lr = ReduceLROnPlateau(factor=0.2, patience=5, min_lr=0.00001, verbose=1)

epochs = 200
batch_size = 32

history = model.fit(X_train, Y_train,
                    validation_split=0.1,
                    epochs=epochs,
                    batch_size=batch_size,
                    callbacks=[early_stopping, model_checkpoint, reduce_lr],
                    verbose=1)


#
# bsh.py
#
# Copyright © 2004-2013, 2016 Monotype Imaging Inc. All Rights Reserved.
#

"""
Support for reading and generating binary-search headers, as are used in
several places in TrueType.
"""

# System imports
import logging

# Other imports
from fontio3 import utilitiesbackend
from fontio3.fontdata import simplemeta

# -----------------------------------------------------------------------------

#
# Constants
#

ERR_CODES = ('V0654', 'V0655', 'V0656')
ERR_STRINGS = ("searchRange", "entrySelector", "rangeShift")

# -----------------------------------------------------------------------------

#
# Functions
#

def binsearchheader(n, unitsize, **kwArgs):
    """
    Returns a string with 4 16-bit binary values: nUnits, searchRange,
    entrySelector and rangeShift.

    >>> utilities.hexdump(binsearchheader(200, 6))
           0 | 00C8 0300 0007 01B0                      |........        |

    >>> utilities.hexdump(binsearchheader(200, 6, use32Bits=True))
           0 | 0000 00C8 0000 0300  0000 0007 0000 01B0 |................|
    """

    kwArgs['skipUnitSize'] = True
    return BSH(nUnits=n, unitSize=unitsize).binaryString(**kwArgs)

# -----------------------------------------------------------------------------

#
# Classes
#

if 0:
    def __________________(): pass

class BSH(object, metaclass=simplemeta.FontDataMetaclass):
    """
    Objects representing binary-search headers. These are simple objects with
    two attributes, nUnits and unitSize. There are also three read-only
    properties: searchRange, entrySelector, and rangeShift.

    >>> bsh = BSH(nUnits=10, unitSize=6)
    >>> utilities.hexdump(bsh.binaryString())
           0 | 0006 000A 0030 0003  000C                |.....0....      |

    >>> bsh.nUnits = 11
    >>> utilities.hexdump(bsh.binaryString())
           0 | 0006 000B 0030 0003  0012                |.....0....      |

    >>> bsh.nUnits = 0
    >>> utilities.hexdump(bsh.binaryString())
           0 | 0006 0000 0006 0000  0000                |..........      |
    """

    #
    # Class definition variables
    #

    attrSpec = dict(
        unitSize = dict(
            attr_initfunc = (lambda: 2),
            attr_label = "Bytes per unit"),

        nUnits = dict(
            attr_initfunc = (lambda: 1),
            attr_label = "Number of units"),

        _cachedKey = dict(
            attr_ignoreforcomparisons = True),

        _cachedFields = dict(
            attr_ignoreforcomparisons = True))

    attrSorted = ('unitSize', 'nUnits')

    #
    # Methods
    #

    def _getEntrySelector(self):
        """
        >>> bsh = BSH(nUnits=10, unitSize=6)
        >>> print((bsh._getEntrySelector()))
        3
        """
        self._recalc()
        return self._cachedFields[1]

    def _getRangeShift(self):
        """
        >>> bsh = BSH(nUnits=10, unitSize=6)
        >>> print((bsh._getRangeShift()))
        12
        """
        self._recalc()
        return self._cachedFields[2]

    def _getSearchRange(self):
        """
        >>> bsh = BSH(nUnits=0, unitSize=6)
        >>> print((bsh._getSearchRange()))
        6
        """
        self._recalc()
        return self._cachedFields[0]

    entrySelector = property(_getEntrySelector)
    rangeShift = property(_getRangeShift)
    searchRange = property(_getSearchRange)

    def _recalc(self):
        """
        Recalculates (if necessary) the three derived fields of this binary
        search header.
        """

        k = (self.unitSize, self.nUnits)

        if k != self._cachedKey:
            # the following line uses an exceedingly hacky
            # way of computing the base 2 log!
            es = max(0, [k[1] < 2**x for x in range(32)].index(1) - 1)
            sr = (2 ** es) * k[0]
            rs = k[0] * max(0, k[1] - (2 ** es))
            self._cachedFields = (sr, es, rs)
            self._cachedKey = k

    def buildBinary(self, w, **kwArgs):
        """
        Adds the binary data for the BSH object to the specified LinkedWriter.
        The following keyword arguments are optional:

            skipUnitSize    If True, the resulting binary data will only
                            comprise nUnits, searchRange, entrySelector, and
                            rangeShift. If False (the default), unitSize will
                            be included (first).

            use32Bits       Default is False. If True, all operations will use
                            32-bit words instead of 16-bit words.

        >>> d = {'use32Bits': True}
        >>> obj = _testingValues[1]
        >>> utilities.hexdump(obj.binaryString())
               0 | 0006 000A 0030 0003  000C                |.....0....      |

        >>> utilities.hexdump(obj.binaryString(**d))
               0 | 0000 0006 0000 000A  0000 0030 0000 0003 |...........0....|
              10 | 0000 000C                                |....            |
        """

        self._recalc()

        use32Bits = kwArgs.get('use32Bits', False)

        if not kwArgs.get('skipUnitSize', False):
            w.add(("L" if use32Bits else "H"), self.unitSize)

        w.add(("4L" if use32Bits else "4H"), self.nUnits, *self._cachedFields)

    @classmethod
    def fromvalidatedwalker(cls, w, **kwArgs):
        """
        Creates and returns a new BSH object from the specified walker, doing
        source validation. The following optional keyword arguments are
        supported:

            autoSkip        Default is True. If True, 5 words will be read from
                            the walker, even though only the first two words
                            (unitSize and nUnits) will be used. If False, only
                            2 words will be read, so the caller must skip or
                            otherwise process the walker for the remaining
                            three words.

            logger          A logger to which messages will be posted.

            recalcFirst     Default is False. If True, the objects three
                            derived values will be recalculated before the
                            object is returned. If False, only nUnits and
                            unitSize will be set.

            use32Bits       Default is False. If True, all operations will use
                            32-bit words instead of 16-bit words.

        >>> FH = utilities.fromhex
        >>> logger = utilities.makeDoctestLogger("bsh_fvw")
        >>> fvb = BSH.fromvalidatedbytes
        >>> obj = fvb(FH("00 06 00 0A 00 30 00 03 00 0C"), logger=logger)
        bsh_fvw.binsrch header - DEBUG - Walker has 10 remaining bytes.
        >>> fvb(FH("00 06 00 0A 00 00 00 00 00 00"), logger=logger)
        bsh_fvw.binsrch header - DEBUG - Walker has 10 remaining bytes.
        bsh_fvw.binsrch header - ERROR - The searchRange should have a value of 48, but is 0.
        bsh_fvw.binsrch header - ERROR - The entrySelector should have a value of 3, but is 0.
        bsh_fvw.binsrch header - ERROR - The rangeShift should have a value of 12, but is 0.

        >>> s = FH("00 00 00 06 00 00 00 0A 00 00 00 30 00 00 00 03 00 00 00 0C")
        >>> obj = fvb(s, logger=logger, use32Bits=True)
        bsh_fvw.binsrch header - DEBUG - Walker has 20 remaining bytes.
        >>> fvb(s[:-1] + FH("99"), logger=logger, use32Bits=True)
        bsh_fvw.binsrch header - DEBUG - Walker has 20 remaining bytes.
        bsh_fvw.binsrch header - ERROR - The rangeShift should have a value of 12, but is 153.

        >>> s = FH("00 00 00 06 ")
        >>> fvb(s[:-1] + FH("99"), logger=logger, use32Bits=True)
        bsh_fvw.binsrch header - DEBUG - Walker has 4 remaining bytes.
        bsh_fvw.binsrch header - ERROR - Insufficient bytes.

        >>> fvb(s[:-1] + FH("99"), logger=logger, use32Bits=True, autoSkip=False)
        bsh_fvw.binsrch header - DEBUG - Walker has 4 remaining bytes.
        bsh_fvw.binsrch header - ERROR - Insufficient bytes.

        >>> a = FH("00 00 00 06 10 00 00 00 00 01 02 00 ")
        >>> x = fvb(a[:-1] + FH("99"), logger=logger, use32Bits=True, autoSkip=False)
        bsh_fvw.binsrch header - DEBUG - Walker has 12 remaining bytes.
        >>> x.nUnits, x.unitSize
        (268435456, 6)

        >>> b = FH("00 00 00 06 10 00 00 00 00 01 02 00 ")
        >>> x = fvb(b[:-1] + FH("99"), logger=logger, use32Bits=False, autoSkip=False)
        bsh_fvw.binsrch header - DEBUG - Walker has 12 remaining bytes.
        >>> x.nUnits, x.unitSize
        (6, 0)

        >>> c = FH("00 00 00 06 ")
        >>> fvb(c[:-1] + FH("99"), logger=logger, use32Bits=False, autoSkip=True)
        bsh_fvw.binsrch header - DEBUG - Walker has 4 remaining bytes.
        bsh_fvw.binsrch header - ERROR - Insufficient bytes.

        >>> d = FH("00 00 06 ")
        >>> fvb(d[:-1] + FH("99"), logger=logger, use32Bits=False, autoSkip=False)
        bsh_fvw.binsrch header - DEBUG - Walker has 3 remaining bytes.
        bsh_fvw.binsrch header - ERROR - Insufficient bytes.
        """

        logger = kwArgs.pop('logger', logging.getLogger())
        logger = logger.getChild("binsrch header")

        logger.debug((
          'V0001',
          (w.length(),),
          "Walker has %d remaining bytes."))

        use32Bits = kwArgs.get('use32Bits', False)
        autoSkip = kwArgs.get('autoSkip', True)

        if use32Bits:
            if autoSkip:
                if w.length() < 20:
                    logger.error(('V0004', (), "Insufficient bytes."))
                    return None

                t = w.unpack("2L")
                parts = w.unpack("3L")

            else:
                if w.length() < 8:
                    logger.error(('V0004', (), "Insufficient bytes."))
                    return None

                t = w.unpack("2L")
                parts = None

        else:
            if autoSkip:
                if w.length() < 10:
                    logger.error(('V0004', (), "Insufficient bytes."))
                    return None

                t = w.unpack("2H")
                parts = w.unpack("3H")

            else:
                if w.length() < 4:
                    logger.error(('V0004', (), "Insufficient bytes."))
                    return None

                t = w.unpack("2H")
                parts = None

        r = cls(*t)

        if parts is not None:
            r._recalc()
            it = zip(parts, r._cachedFields, ERR_CODES, ERR_STRINGS)

            for part, good, code, s in it:
                if part != good:
                    if all(t):
                        logger.error((
                          code,
                          (s, good, part),
                          "The %s should have a value of %d, but is %d."))

                        r = None

                    else:
                        logger.warning((
                          code,
                          (s, good, part),
                          "The %s should have a value of %d, but is %d. "
                          "However, since nUnits and/or unitSize are zero, "
                          "this is only a warning."))

        return r

    @classmethod
    def fromwalker(cls, w, **kwArgs):
        """
        Creates and returns a new BSH object from the specified walker. The
        following optional keyword arguments are supported:

            autoSkip        Default is True. If True, 5 words will be read from
                            the walker, even though only the first two words
                            (unitSize and nUnits) will be used. If False, only
                            2 words will be read, so the caller must skip or
                            otherwise process the walker for the remaining
                            three words.

            recalcFirst     Default is False. If True, the objects three
                            derived values will be recalculated before the
                            object is returned. If False, only nUnits and
                            unitSize will be set.

            use32Bits       Default is False. If True, all operations will use
                            32-bit words instead of 16-bit words.

        >>> d = {'use32Bits': True}
        >>> obj = _testingValues[1]
        >>> obj == BSH.frombytes(obj.binaryString())
        True
        >>> obj == BSH.frombytes(obj.binaryString(**d), **d)
        True
        """

        use32Bits = kwArgs.get('use32Bits', False)
        r = cls(*w.group(("L" if use32Bits else "H"), 2))

        if kwArgs.get('autoSkip', True):
            w.skip(12 if use32Bits else 6)

        return r

# -----------------------------------------------------------------------------

#
# Test code
#

if 0:
    def __________________(): pass

if __debug__:
    from fontio3 import utilities

    _testingValues = (
        BSH(),
        BSH(nUnits=10, unitSize=6))

def _test():
    import doctest
    doctest.testmod()

if __name__ == "__main__":
    if __debug__:
        _test()


import asyncio
from .taskplanner import TaskPlanner
from os.path import join, isfile, isabs, dirname, abspath
from .common import fix_buffering, get_full_url
import argparse
import pickle
from sys import stderr

__all__ = ['run_scrape', 'request', 'challenge', 'go_parallel', 'get_loop']

parser = argparse.ArgumentParser()
parser.add_argument('--pipe', required=False, help='For inner usage by monitor process')
parser.add_argument('--noresume', action='store_true', help='Don\'t auto resume on start')
parser.add_argument('--verbose', action='store_true', help='Dump all messages intended for monitor')

fix_buffering()

task_planner = None

def get_loop():
    return task_planner.loop

async def go_parallel(corolist):
    if isinstance(corolist, dict):
        tmap, tasks = {}, []
        for k, coro in corolist.items():
            t = asyncio.ensure_future(coro, loop=task_planner.loop)
            tmap[id(t)] = k
            tasks.append(t)
    else:
        tasks = [asyncio.ensure_future(coro, loop=task_planner.loop) for coro in corolist]
    for t in tasks:
        t._log_destroy_pending = False
    done, _ = await asyncio.wait(tasks, loop=task_planner.loop)
    result = {} if isinstance(corolist, dict) else []
    for task in done:
        e = task.exception()
        if e is not None:
            print('go_parallel:', e, type(e), file=stderr)
        else:
            if isinstance(corolist, dict):
                result[tmap[id(task)]] = task.result()
            else:
                result.append(task.result())
    return result

async def _parallel_worker(coroiter, output):
    while True:
        try:
            try:
                key, coro = next(coroiter)
            except StopIteration:
                return
            output[key] = await coro
        except Exception as e:
            print('go_parallel:', e, type(e), file=stderr)

async def go_parallel_limited(corolist, workers=50):
    'Replacement for go_parallel with controllable memory consumption'

    dictmode = True
    if isinstance(corolist, dict):
        keycoroiter = iter(corolist.items())
    else:
        keycoroiter = enumerate(corolist)
        dictmode = False
    output = {}
    tasks = [
        asyncio.ensure_future(_parallel_worker(keycoroiter, output), loop=task_planner.loop)
        for i in range(min(workers, len(corolist)))
    ]
    for t in tasks:
        t._log_destroy_pending = False
    await asyncio.wait(tasks, loop=task_planner.loop)
    if dictmode:
        return output
    else:
        result = []
        for i in range(len(corolist)):
            if i not in output:
                continue
            result.append(output[i])
        return result

async def request(
    *,
    method='get', url, params=None, data=None, headers=None, auth=None, cookies=None,
    key=None, parser=None, parserkw=None,
    force_request_queue=None
):
    if params is not None:
        url = get_full_url(url, params)
    req = {
        'method': method,
        'url': url,
    }
    for k, v in (
        ('data', data),
        ('headers', headers),
        ('auth', auth),
        ('cookies', cookies),
        ('parserkw', parserkw),
        ('key', key),
        ('parser', parser),
    ):
        if v is not None:
            req[k] = v
    return (await task_planner.work_over_full_line(req, force_request_queue=force_request_queue))

async def challenge(html):
    return (await task_planner.challenge(html))

# TODO: prepare not a session, but domain
# i.e. we need authorization only for particular domain

async def control_reader_worker(reader, queue):
    # TODO: duplicate
    while True:
        msglen = await reader.readexactly(4)
        msglen = int.from_bytes(msglen, byteorder='big', signed=False)
        msg = await reader.readexactly(msglen)
        msg = pickle.loads(msg)
        await queue.put(msg)

async def control_writer_worker(writer, queue):
    # TODO: duplicate
    while True:
        msg = await queue.get()
        msg = pickle.dumps(msg)
        msglen = len(msg).to_bytes(length=4, byteorder='big')
        writer.write(msglen)
        writer.write(msg)
        await writer.drain()

async def null_queue_getter(queue, dump):
    while True:
        msg = await queue.get()
        if dump:
            print(msg)
        elif msg.get('type') == 'parserStdout':
            print(msg.get('line', '').rstrip('\n'))
        elif msg.get('type') == 'parserStderr':
            print(msg.get('line', '').rstrip('\n'), file=stderr)

async def scrape_main(main, prepare_session, parser_filename, config, *, proxy_list=None, sockfile='scutigera', loop):
    global task_planner

    args = parser.parse_args()

    rq = asyncio.Queue(loop=loop)
    wq = asyncio.Queue(loop=loop)

    if args.pipe is not None:
        reader, writer = await asyncio.open_unix_connection(args.pipe, loop=loop)
        t = asyncio.ensure_future(control_reader_worker(reader, rq), loop=loop)
        t._log_destroy_pending = False
        t = asyncio.ensure_future(control_writer_worker(writer, wq), loop=loop)
        t._log_destroy_pending = False
    else:
        t = asyncio.ensure_future(null_queue_getter(wq, args.verbose), loop=loop)
        t._log_destroy_pending = False
    del t

    task_planner = TaskPlanner(
        config, parser_filename,
        prepare_session=prepare_session, sockfile=sockfile,
        r_control=rq, w_control=wq, proxy_list=proxy_list,
        loop=loop
    )
    if args.pipe is None and not args.noresume:
        # if not connected to master server go on its own
        asyncio.ensure_future(task_planner.resume(), loop=loop)
    try:
        await main()
    finally:
        await asyncio.sleep(1)  # prevent finishing too fast, e.g. if main is a plain func
    await task_planner.finalize()

def load_proxylist(fname):
    proxy_list = []
    if not isfile(fname):
        return proxy_list
    with open(fname) as f:
        for line in f:
            line = line.strip()
            if not line:
                continue
            v = line.split(' ')
            addr = v[0]
            auth = (v[1], v[2]) if len(v) >= 3 else None
            proxy_list.append((addr, auth))
    return proxy_list

def run_scrape(main, prepare_session, parser_modulename, config):
    loop = asyncio.get_event_loop()

    proxy_list = []
    if config.get('AllowProxy') is True:
        proxfname = config.get('ProxyListFile')
        if not isabs(proxfname):
            proxfname = join(dirname(dirname(abspath(__file__))), proxfname)
        print(proxfname)
        proxy_list = load_proxylist(proxfname)
        print(len(proxy_list), 'proxies')

    loop.run_until_complete(scrape_main(
        main, prepare_session, parser_modulename, config, proxy_list=proxy_list, loop=loop
    ))
    loop.close()


# Copyright 2018-Present Shanghai Yitu Technology Co., Ltd.
# Licensed under the Apache License, Version 2.0

import base64
import os
import hashlib

# Converts file into base64 encoded string
def load_file_base64(file_path):
    if not os.path.isfile(file_path):
        return None

    with open(file_path, "rb") as audio_file:
        encoded_string = base64.b64encode(audio_file.read())
        return encoded_string.decode('utf-8')

def md5sum(filename):
    #python3
    fd = open(filename, "rb")
    #python2
    # fd = open(filename, "rb")
    fcont = fd.read()
    fd.close()
    return hashlib.md5(fcont).hexdigest()


# -*- coding: utf-8 -*-
# Copyright (c) 2016 Aladom SAS & Hosting Dvpt SAS
from datetime import datetime
import os
import re

from django.core.signing import Signer
from django.urls import reverse
from django.db import models
from django.template import Template
from django.template.loader import get_template
from django.utils import timezone
from django.utils.translation import ugettext_lazy as _

from ..conf import (
    TextConfRef, TEMPLATES_UPLOAD_DIR, SUBJECT_PREFIX, MIRROR_SIGNING_SALT,
)
from .manager import BlacklistManager, SubscriptionManager
from .options import (
    AbstractBaseMailHeader, AbstractBaseStaticAttachment,
    AbstractBaseDynamicAttachment,
)

__all__ = [
    'Campaign', 'CampaignMailHeader', 'CampaignStaticAttachment',
    'Mail', 'MailHeader', 'MailStaticAttachment', 'MailDynamicAttachment',
    'SubscriptionType', 'Subscription', 'Blacklist',
]

def templates_upload_to(instance, filename):
    if callable(TEMPLATES_UPLOAD_DIR):
        return TEMPLATES_UPLOAD_DIR(instance, filename)
    else:
        return os.path.join(format(datetime.now(), TEMPLATES_UPLOAD_DIR),
                            filename)

class SubscriptionType(models.Model):

    class Meta:
        verbose_name = _("subscription type")
        verbose_name_plural = _("subscription types")

    name = models.CharField(
        max_length=200, unique=True, verbose_name=_("name"))
    description = models.TextField(verbose_name=_("description"))
    subscribed_by_default = models.BooleanField(
        default=True, verbose_name=_("subscribed by default"),
        help_text=_(
            "Whether potential recipients are subscribed or not to "
            "this type by default"
        ))

    def __str__(self):
        return self.name

    def is_subscribed(self, email):
        match = re.match(r'.*\s<([^<> ]+)>', email)
        if match:
            email = match.group(1)

        try:
            return self.subscriptions.get(email=email).subscribed
        except Subscription.DoesNotExist:
            return self.subscribed_by_default

class Subscription(models.Model):

    class Meta:
        verbose_name = _("subscription")
        verbose_name_plural = _("subscriptions")
        unique_together = [
            ('email', 'subscription_type'),
        ]

    email = models.EmailField(
        db_index=True, verbose_name=_("e-mail"))
    subscription_type = models.ForeignKey(
        SubscriptionType, models.CASCADE,
        related_name='subscriptions', related_query_name='subscriptions',
        verbose_name=_("subscription type"))
    subscribed = models.BooleanField(
        default=True, verbose_name=_("subscribed"))
    date_joined = models.DateTimeField(default=timezone.now)
    last_modified = models.DateTimeField(default=timezone.now)

    objects = SubscriptionManager()

    def save(self, *args, **kwargs):
        self.last_modified = timezone.now()
        super().save(*args, **kwargs)

class Campaign(models.Model):

    class Meta:
        ordering = ['name']
        verbose_name = _("e-mail campaign")
        verbose_name_plural = _("e-mail campaigns")

    key = models.SlugField(
        max_length=50, unique=True, verbose_name=_("key"),
        help_text=_(
            "The key will be used to reference this campaign from your "
            "scripts."
        ))
    name = models.CharField(
        max_length=255, verbose_name=_("name"))
    subject = models.CharField(
        max_length=255, verbose_name=_("e-mail subject"),
        help_text=_("May contain template variables."))
    prefix_subject = models.BooleanField(
        default=True, verbose_name=_("prefix subject"),
        help_text=TextConfRef('SUBJECT_PREFIX', within=_(
            "Wheter to prefix the subject with \"{}\" or not."
        )))
    is_enabled = models.BooleanField(
        default=True, verbose_name=_("enabled"),
        help_text=_(
            "E-mails won't be sent for campaigns that are not enabled. "
            "Even if a script requests for sending. This is a way to turn off "
            "some campaigns temporarily without changing the source code."
        ))
    debug_mode = models.BooleanField(
        default=False, verbose_name=_("debug mode"))
    template_file = models.FileField(
        upload_to=templates_upload_to, blank=True,
        verbose_name=_("template file"),
        help_text=_(
            "Leave blank to use mailing/{key}.html "
            "from within your template directories."
        ))
    subscription_type = models.ForeignKey(
        SubscriptionType, models.SET_NULL, blank=True, null=True,
        verbose_name=_("subscription type"))

    def __str__(self):
        return self.key

    def get_template(self):
        if self.template_file:
            with open(self.template_file.path, 'r') as f:
                template = Template(f.read())
        else:
            template = get_template('mailing/{key}.html'.format(key=self.key))
        return template

    def get_subject(self):
        if self.prefix_subject and SUBJECT_PREFIX:
            return '{} {}'.format(SUBJECT_PREFIX, self.subject)
        else:
            return self.subject

    def is_subscribed(self, email):
        return (not self.subscription_type or
                self.subscription_type.is_subscribed(email))

class CampaignMailHeader(AbstractBaseMailHeader):

    class Meta:
        verbose_name = _("extra header")
        verbose_name_plural = _("extra headers")

    campaign = models.ForeignKey(
        'Campaign', models.CASCADE, related_name='extra_headers')

class CampaignStaticAttachment(AbstractBaseStaticAttachment):

    campaign = models.ForeignKey(
        'Campaign', models.CASCADE, related_name='static_attachments')

class Mail(models.Model):

    class Meta:
        ordering = ['-scheduled_on']
        verbose_name = _("e-mail")
        verbose_name_plural = _("e-mails")

    STATUS_PENDING = 1
    STATUS_SENT = 2
    STATUS_CANCELED = 3
    STATUS_FAILURE = 4
    STATUS_DRAFT = 5
    STATUS_CHOICES = [
        (STATUS_PENDING, _("Pending")),
        (STATUS_SENT, _("Sent")),
        (STATUS_CANCELED, _("Canceled")),
        (STATUS_FAILURE, _("Failure")),
        (STATUS_DRAFT, _("Draft")),
    ]

    campaign = models.ForeignKey(
        'Campaign', models.SET_NULL, blank=True, null=True,
        verbose_name=_("campaign"))
    status = models.PositiveSmallIntegerField(
        choices=STATUS_CHOICES, default=STATUS_DRAFT,
        verbose_name=_("status"))
    scheduled_on = models.DateTimeField(
        default=timezone.now, verbose_name=_("scheduled on"))
    sent_on = models.DateTimeField(
        blank=True, null=True, editable=False, verbose_name=_("sent on"))
    subject = models.CharField(
        max_length=255, verbose_name=_("subject"))
    html_body = models.TextField(
        verbose_name=_("HTML body"))
    text_body = models.TextField(
        blank=True, verbose_name=_("text body"),
        help_text=_("Leave blank to generate from HTML body."))
    failure_reason = models.TextField(
        blank=True, editable=False, verbose_name=_("failure reason"))

    def __str__(self):
        return "[{}] {}".format(self.scheduled_on, self.subject)

    def get_headers(self):
        headers = dict(self.headers.items())
        headers.setdefault("X-Mail-Id", str(self.pk))
        return headers

    def get_attachments(self):
        return list(self.static_attachments.all()) + list(self.dynamic_attachments.all())

    def get_absolute_url(self):
        signer = Signer(salt=MIRROR_SIGNING_SALT)
        signed_pk = signer.sign(str(self.pk))
        return reverse('mailing:mirror', kwargs={'signed_pk': signed_pk})

class MailHeader(AbstractBaseMailHeader):

    class Meta:
        verbose_name = _("header")
        verbose_name_plural = _("headers")

    mail = models.ForeignKey(
        'Mail', models.CASCADE, related_name='headers')

class MailStaticAttachment(AbstractBaseStaticAttachment):

    mail = models.ForeignKey(
        'Mail', models.CASCADE, related_name='static_attachments')

class MailDynamicAttachment(AbstractBaseDynamicAttachment):

    mail = models.ForeignKey(
        'Mail', models.CASCADE, related_name='dynamic_attachments')

class Blacklist(models.Model):

    class Meta:
        verbose_name = _("blacklisted address")
        verbose_name_plural = _("blacklisted addresses")
        ordering = ['-reported_on']

    REASON_OTHER = 1
    REASON_SPAM = 2
    REASON_BLOCKED = 3
    REASON_HARDBOUNCE = 4
    REASON_CHOICES = [
        (REASON_SPAM, _("SPAM")),
        (REASON_BLOCKED, _("Blocked")),
        (REASON_HARDBOUNCE, _("Hard bounce")),
        (REASON_OTHER, _("Other")),
    ]

    email = models.EmailField(
        unique=True, verbose_name=_("e-mail address"))
    reason = models.PositiveSmallIntegerField(
        choices=REASON_CHOICES, default=REASON_OTHER,
        verbose_name=_("reason"))
    verbose_reason = models.CharField(
        max_length=250, blank=True, verbose_name=_("verbose reason"))
    reported_on = models.DateTimeField(
        verbose_name=_("reported on"), auto_now_add=True)

    objects = BlacklistManager()

    def __str__(self):
        return "{} ({})".format(self.email, self.get_reason_display())


import sys  # связь с ОС
from PyQt5 import QtWidgets
import design

class ChatWindow(QtWidgets.QMainWindow, design.Ui_MainWindow):
    def __init__(self):
        super().__init__()
        self.setupUi(self)
        self.init_handlers()

    def init_handlers(self):  # инициализация обработчиков
        self.pushButton.clicked.connect(self.send_message)

    def send_message(self):
        message = self.lineEdit.text()
        self.plainTextEdit.appendPlainText(message)
        self.lineEdit.clear()

app = QtWidgets.QApplication(sys.argv)
window = ChatWindow()
window.show()  # показ окна
app.exec_()  # запуск интерфейса


import argparse

# constant
__version__ = '1.0'
__description__ = 'schat - a simple security multiple users chat room '
__epilog__ = 'Report bugs to <cjyeh@cs.nctu.edu.tw>'

def parse_argv():
	# init
	parser = argparse.ArgumentParser(
		description=__description__,
		epilog=__epilog__
	)

	# main parser
	parser.add_argument(
		'-v', '-V', '--version',
		action='version',
		help='Print program version',
		version='v{}'.format(__version__)
	)

	# sub parser
	subparsers = parser.add_subparsers(
		dest='command',
		help='command help'
	)
	subparsers.required = True

	# server
	parser_server = subparsers.add_parser('server')
	parser_server.add_argument(
		'-p', '--port',
		action='store',
		help='Specified port, default=7000',
		type=int,
		default=7000
	)
	parser_server.add_argument(
		'-C', '--capacity',
		action='store',
		help='Maximum capacity for chatroom, default=10',
		type=int,
		default=10
	)
	parser_server.add_argument(
		'-k', '--key',
		action='store',
		help='Private key'
	)
	parser_server.add_argument(
		'-c', '--cert',
		action='store',
		help='Certification'
	)

	# client
	parser_client = subparsers.add_parser('client')
	parser_client.add_argument('host')
	parser_client.add_argument(
		'-p', '--port',
		action='store',
		help='Specified port, default=7000',
		type=int,
		default=7000
	)
	parser_client.add_argument(
		'-s', '--secure',
		action='store_true',
		help='Secure connection',
		default=False
	)
	parser_client.add_argument(
		'-v', '-V', '--version',
		action='version',
		help='Print program version',
		version='v{}'.format(__version__)
	)

	results = parser.parse_args()

	return results.__dict__


# -*- coding: utf-8 -*-

import json
from django.contrib.auth.models import User
from django.http import HttpResponse
from django.views.decorators.csrf import csrf_exempt
# Create your views here.
from godsome.main.models import get_or_none
from godsome.message.models import MobileDevice

@csrf_exempt
def push(request):
    state = []
    uid = request.GET.get("uid")
    regid = request.GET.get("regid")
    os = request.GET.get("os",0)
    user = get_or_none(User,id=uid)
    if regid and user :
        md = get_or_none(MobileDevice,uid=regid)
        if md :
            md.user = user
            md.os = os
        else :
            MobileDevice.objects.get_or_create(user=user,uid=regid,os=os)
    state.append(regid)
    return HttpResponse(json.dumps(state), content_type="application/json")


import sys
import openpyxl
from os import path

N = int(sys.argv[1])
M = int(sys.argv[2])
NAME = sys.argv[3]

wb = openpyxl.load_workbook(NAME)
sheet = wb.get_active_sheet()

sheet.insert_rows(N, M)

wb.save(path.join('.', 'dist', path.basename(NAME)))


# -*- coding: utf-8 -*-
import logging
from binascii import hexlify

from cryptoauthlib.constant import INFO_MODE_REVISION, INFO_MODE_STATE
from cryptoauthlib.device import ATECC508A

log = logging.getLogger("atecc508a.tests_info")

def run(atecc508a=None):
    if not atecc508a:
        atecc508a = ATECC508A()

    expected = b'\x07\x00\x00P\x00\x03\x91'
    packet = atecc508a.atcab_info()
    assert expected == packet.response_data
    log.debug("atcab_info: %s", hexlify(packet.response_data))

    packet = atecc508a.atcab_info_base(INFO_MODE_REVISION)
    assert expected == packet.response_data
    log.debug("atcab_info_base - revision: %s", hexlify(packet.response_data))

    expected = b'\x07\x00\x00\x00\x00\x03\xad'
    packet = atecc508a.atcab_info_base(INFO_MODE_STATE)
    assert expected == packet.response_data
    log.debug("atcab_info_base - state: %s", hexlify(packet.response_data))


"""
CLI for `mirror github`
"""

import argparse
from typing import Callable, Dict

from . import allrepos, licenses, sync
from ..populate import populate_cli

subcommand = 'github'

SUBCOMMAND_POPULATORS: Dict[str, Callable[[argparse.ArgumentParser], None]] = {
    'crawl': allrepos.crawl_populator,
    'nextid': allrepos.nextid_populator,
    'sample': allrepos.sample_populator,
    'validate': allrepos.validate_populator,
    'sync': sync.populator,
    licenses.subcommand: licenses.populator,
}

def populator(parser: argparse.ArgumentParser) -> None:
    populate_cli(parser, SUBCOMMAND_POPULATORS)


import logging
import time as t

from setup import PROXY, TOKEN
from telegram.ext import CallbackContext, CommandHandler, Filters, MessageHandler, Updater, ConversationHandler, CallbackQueryHandler
from telegram import Bot, Update, InlineKeyboardMarkup, InlineKeyboardButton

from nltk.corpus import stopwords
from pymystem3 import Mystem
from string import punctuation
import json

mystem = Mystem()
russian_stopwords = stopwords.words("russian") + ["ко"]

bot = Bot(
        token=TOKEN,
        base_url=PROXY,  # delete it if connection via VPN
    )

def start(update: Update, context: CallbackContext):
    """Send a message when the command /start is issued."""
    text = f'Hello, {update.effective_user.first_name}! 👋\nWelcome to the smart bot that is able to detect and extract key words from text!\n Write /help to learn detailes.'
    update.message.reply_text(text)
    return text

def help(update: Update, context: CallbackContext):
    update.message.reply_text("In order to find key words in your document it is required that you send me a message containing document only. Make sure there are no other charachters. Preferred type of doc is .txt. Hope you'll enjoy!")

def get_file(update: Update, context: CallbackContext):
    print("get file function is started")
    overall_start = t.time()
    document = update["message"]["document"]["file_id"]
    file = context.bot.getFile(file_id=document)
    file.download("file.txt")
    key_words, tokens = keywords()
    overall_finish = t.time()
    update.message.reply_text(f"According to my findings, your file containes {len(key_words)} significant words.")
    update.message.reply_text(f"It took {overall_finish - overall_start} seconds.")
    update.message.reply_text(f"The length of your doc is {len(tokens)} words")
    update.message.reply_text(" ".join(key_words[:10]))

def keywords():
    print("key words is started")
    # making a list of all lemmas from the doc
    start = t.time()
    with open("file.txt", encoding="utf8") as doc:
        tokens = mystem.lemmatize(doc.read().lower())
        tokens = [token for token in tokens if token not in russian_stopwords \
                  and token != " " \
                  and token.strip() not in punctuation]
        tokens = [token for token in tokens if token.isalpha() and len(token) > 1]
    finish = t.time()
    print(f"the doc is lemmatized in {start - finish} secs")

    #opening the json with all the frequencies of words from NCRL and counting the length of the corpora
    start = t.time()
    with open("frequencies.json") as doc:
        corpora = json.load(doc)
        corp_total = 0
        for i in corpora:
            corp_total += corpora[i]
    finish = t.time()
    print(f"frequencies.json are open and ready to be used in {start - finish} secs")
    print(corpora["солнце"])

    start = t.time()
    doc_frqn = {}
    corp_frqn = {}

    for token in tokens:
        doc_frqn[token] = tokens.count(token) / len(tokens)
        try:
            corp_frqn[token] = corpora[token] / corp_total
        except KeyError:
            corp_frqn[token] = 0
    finish = t.time()
    print(f"numbers of entrances is calculated succesfully in {finish - start} secs")

    expected = {}
    for token in doc_frqn.keys():
        a = doc_frqn[token]
        b = corp_frqn[token]
        c = 1 - a
        d = 1 - b
        expected[token] = ((a + b) * (a + c)) / (a + b + c + d)

    print("expected frequency is calculated successfully ")

    chi = {}
    for token in expected.keys():
        o = doc_frqn[token]
        e = expected[token]
        chi[token] = ((o - e) ** 2) / e

    l = list(chi.items())
    l.sort(key=lambda i: i[1])
    l = l[::-1]
    print("chi2 value is calculated successfully")
    print(l)

    Q = 0.003932

    kw = []
    for i in l:
        if i[1] <= Q:
            kw.append(i[0])
    print("keywords list is formed")
    print("keywords function is finished")
    print(" ".join(kw))
    return(kw, tokens)

def main():
    bot = Bot(
        token=TOKEN,
        base_url=PROXY,  # delete it if connection via VPN
    )
    updater = Updater(bot=bot, use_context=True)
    updater.dispatcher.add_handler(MessageHandler(Filters.document, get_file))
    updater.dispatcher.add_handler(CommandHandler('start', start))
    updater.dispatcher.add_handler(CommandHandler('help', help))

    updater.start_polling()

    # Run the bot until you press Ctrl-C or the process receives SIGINT,
    # SIGTERM or SIGABRT. This should be used most of the time, since
    # start_polling() is non-blocking and will stop the bot gracefully.
    updater.idle()

if __name__ == '__main__':
    print("Bot is started")
    main()