import numpy as np
import entropy
import pandas as pd

from pathlib import Path
from helper_functions import convert_assignment_to_clusters as convert_cl

from dhillion import divisive_cluster
import argparse
from all_algos import algo1, algo2, algo3
import time
import warnings
warnings.filterwarnings("ignore", category=np.VisibleDeprecationWarning)

if __name__ == "__main__":
    # pass argument and checking
    parser = argparse.ArgumentParser(description='Argument for R(e_max) algorithm')
    parser.add_argument('-filename',
                        type=str,
                        help='Input filename')

    # parse arg
    args = parser.parse_args()
    filename = Path(args.filename+".csv")
    # filename = Path("ng20.csv")

    data = np.genfromtxt(filename, delimiter=',')
    m, n = data.shape
    # k = args.K
    print("Dimenions of data: %d, %d" %(m, n))

    k_list = list(range(2,20)) + list(range(20, 51, 10)) + [100,200,500,1000,2000]

    e_list = []
    R_e_list = []
    nguyen_impurity_list = []
    nguyen_time_list = []

    dhllion_impurity_list = []
    dhllion_time_list = []

    for k in k_list:
    # nguyen 2021
        nguyen_begin = time.time()
        assnment = algo1(data)
        partitions = convert_cl(assnment, data)

        if len(partitions) == k:
            clusters = partitions
        elif len(partitions) > k:
            clusters = algo2(len(partitions), k, partitions)
        else:
            clusters = algo3(len(partitions), k, partitions)
        nguyen_end = time.time()

        nguyen_time = nguyen_end - nguyen_begin
        impurity_nguyen, e, R_e = entropy.partition_entropy(partitions, data, converted = True)

        e_list.append(e)
        R_e_list.append(R_e)
        nguyen_impurity_list.append(impurity_nguyen)
        nguyen_time_list.append(nguyen_time)

        # dhillion et al
        s = np.random.randint(low=1, high=100, size=1)[0]
        dhillion_begin = time.time()
        dc_dhillion = divisive_cluster(init_type = "dhillion", data = data, k = k, seed = s)
        dc_dhillion.k_means()
        dhillion_end = time.time()
        dhillion_time = dhillion_end - dhillion_begin
        impurity_dhillion = dc_dhillion.get_impurity()

        dhllion_impurity_list.append(impurity_dhillion)
        dhllion_time_list.append(dhillion_time)

        print("K = ", k)
        print("Nguyen et. al. result R(e): {R_e:.2f}, e: {e:.2f}, entropy: {impurity:.2f}, taking {time:.2f}(s)".format(R_e = R_e, e = e, impurity = impurity_nguyen, time = nguyen_time))
        print("Dhillion et. al. result entropy: {impurity:.2f}, taking {time:.2f}(s)".format(impurity = impurity_dhillion, time = dhillion_time))

    df = pd.DataFrame(data = {'k':k_list, 'e': e_list, 'R(e)':R_e_list, \
                'entropy_nguyen': nguyen_impurity_list, "nguyen_time":nguyen_time_list,\
                'dhllion_nguyen': dhllion_impurity_list, "dhllion_time":dhllion_time_list})
    name = args.filename+"_entropies.csv"
    df.to_csv(name, index = False)


#!/usr/bin/env python3

from xml.etree import ElementTree
import os.path
import glob
import functools
import itertools

ROOT = 'doc/xml/'
OUTPUT_ROOT = 'build/linker/'

@functools.lru_cache(maxsize=None)
def load_xml_file(path):
    tree = ElementTree.parse(path)
    return tree.getroot()

def load_member(file_refid, member_refid):
    root = load_xml_file(os.path.join(ROOT, file_refid + '.xml'))
    return root.find(".//memberdef[@id='{}']".format(member_refid))

def load_by_ref_id(refid):
    root = load_xml_file(os.path.join(ROOT, 'index.xml'))
    compound = root.find(".//member[@refid='{}']..".format(refid))
    return load_member(compound.attrib['refid'], refid)

def build_entry(member):
    address = next(
        filter(
            lambda el: el.attrib['id'].startswith('address'),
            member.findall('.//xrefsect')
        )
    )

    kind = member.attrib['kind']

    if kind not in ['function', 'variable']:
        raise RuntimeError('Cannot have address for a ' + kind)

    return {
        'name': member.find('name').text,
        'file': member.find('location').attrib['file'],
        'kind': kind,
        'defintion': member.find('definition').text,
        'address': int(address.find('xrefdescription/para').text.strip(), 16)
    }

def write_entry(entry):
    thumb = entry['kind'] == 'function'
    return '{name} = 0x{address:08x}{extra};'.format(extra='|1' if thumb else '', **entry)

def load_addresses(filename):
    root = load_xml_file(filename)

    # Extract the version code string
    version = root.find('compounddef').attrib['id'].split('address')[-1]

    # Build up a list of entries
    xpath = ".//ref[@kindref='member']"
    entries = [build_entry(load_by_ref_id(ref.attrib['refid'])) for ref in root.findall(xpath)]

    # Write a linker script
    output_path = os.path.join(OUTPUT_ROOT, version + '.ld')
    print('Generating linker script `{}` from `{}` ({} entries)'.format(
        output_path,
        filename,
        len(entries)
    ))

    with open(output_path, 'a') as output:
        keyfn = lambda x: x['file']
        for filename, group in itertools.groupby(sorted(entries, key=keyfn), keyfn):
            print('/*\n * {}\n */\n'.format(filename), file=output)

            for entry in group:
                print(write_entry(entry), file=output)

            print(file=output)

for filename in glob.glob(os.path.join(ROOT, 'address*.xml')):
    load_addresses(filename)


import numpy as np
import pylab as plt
from matplotlib.ticker import MultipleLocator

'''
python plot_aFe_aCF.py
'''

# Output file name
outdir = '../ms/figures/'
fout   = outdir + 'aFe_aCF.eps'

# USE z-SCORE TO CONVERT TO 68% UNCERTAINTY

# Continuum fitting measured black hole spins and errors: [value, -err, +err]
aCF_U1543 = [0.8, 0.1, 0.1]    # 68%...?
aCF_CygX1 = [0.9985, 0.0148, 0.0005] # 68%
aCF_1915  = [0.98]             # 3-sigma
aCF_J1655 = [0.7, 0.1, 0.1]    # 68%...?
aCF_LMCX1 = [0.92, 0.07, 0.05] # 68%
aCF_J1550 = [0.34, 0.28, 0.20] # 68%

# Iron line measured black hole spins and errors: [value, -err, +err]
aFe_U1543 = [0.3, 0.1, 0.1]     # 68%
aFe_CygX1 = [0.97, 0.02, 0.014] # 90%
aFe_1915  = [0.98, 0.01, 0.01]  # 68%
aFe_J1655 = [0.92, 0.02, 0.02]  # 90%
aFe_LMCX1 = [0.97, 0.25, 0.02]  # 68%
aFe_J1550 = [0.55, 0.22, 0.15]  # 68%

# Use z-scores to convert uncertainties to 1-sigma level
z90 = 1.645
aFe_CygX1[1], aFe_CygX1[2] = aFe_CygX1[1]/z90, aFe_CygX1[2]/z90
aFe_J1655[1], aFe_J1655[2] = aFe_J1655[1]/z90, aFe_J1655[2]/z90

# Plotting preferences
dpi = 300
xsize, ysize = 8.4, 8.4
left, right, bottom, top = 0.15, 0.97, 0.15, 0.97
fs, fs_sm, lw, pad, tlmaj, tlmin = 28, 24, 2, 12, 10, 5

# Plotting preferences
ts          = 16.0  # Text size
legfs       = 16.0  # Legend font size
symsize     = 300   # Symbol size
symsize_leg = 150   # Symbol size (for legend)
csize       = 0.0   # Capsize for error bars
csize_arrow = 5.0
elw         = 2.0   # Width of error bars
mew         = 2.0   # Marker edge width
limlen      = 0.1   # Length of x error upper/lower limits arrow

a_min, a_max = -1,  1
xmin,  xmax  = 0.0, 1.1
ymin,  ymax  = 0.0, 1.1
xinc_maj, xinc_min = 0.2, 0.05
yinc_maj, yinc_min = 0.2, 0.05
xmajorLocator = MultipleLocator(xinc_maj)
xminorLocator = MultipleLocator(xinc_min)
ymajorLocator = MultipleLocator(yinc_maj)
yminorLocator = MultipleLocator(yinc_min)

mark_U1543 = 'o'  # circle
mark_CygX1 = '^'  # up-triangle
mark_1915  = 'v'  # down-triangle
mark_J1655 = 's'  # square
mark_LMCX1 = 'D'  # diamond
mark_J1550 = '*'  # star

fig = plt.figure(figsize=(8.4, 8.4), dpi=dpi)
fig.subplots_adjust(left=left, right=right, bottom=bottom, top=top)
ax = fig.add_subplot(111)
ax.axhline(y=ymin, xmin=xmin, xmax=xmax, linewidth=lw, color='k')
ax.axhline(y=ymax, xmin=xmin, xmax=xmax, linewidth=lw, color='k')
ax.axvline(x=xmin, ymin=ymin, ymax=ymax, linewidth=lw, color='k')
ax.axvline(x=xmax, ymin=ymin, ymax=ymax, linewidth=lw, color='k')
ax.set_xlabel(r"${\rm Iron\ Line\ Black\ Hole\ Spin,}\ a^{\rm Fe}$",         fontsize=fs, labelpad=pad)
ax.set_ylabel(r"${\rm Continuum\ Fitting\ Black\ Hole\ Spin,}\ a^{\rm CF}$", fontsize=fs, labelpad=pad)
ax.set_xlim(xmin, xmax)
ax.set_ylim(ymin, ymax)
ax.tick_params('both', direction='in', length=tlmaj, width=lw, which='major', labelsize=fs_sm, pad=pad)
ax.tick_params('both', direction='in', length=tlmin, width=lw, which='minor')
ax.xaxis.set_major_locator(xmajorLocator)
ax.xaxis.set_minor_locator(xminorLocator)
ax.yaxis.set_major_locator(ymajorLocator)
ax.yaxis.set_minor_locator(yminorLocator)

ax.plot([xmin,xmax], [ymin,ymax], 'k--', linewidth=lw)

s_U1543 = ax.scatter(aFe_U1543[0], aCF_U1543[0], marker=mark_U1543, s=symsize, facecolors='w', edgecolors='k', linewidths=elw, zorder=3)
ax.errorbar([aFe_U1543[0],aFe_U1543[0]], [aCF_U1543[0],aCF_U1543[0]], \
            xerr=[[aFe_U1543[1],aFe_U1543[1]], [aFe_U1543[2],aFe_U1543[2]]], \
            yerr=[[aCF_U1543[1],aCF_U1543[1]], [aCF_U1543[2],aCF_U1543[2]]], \
            fmt='none', marker=None, markeredgewidth=mew, capsize=csize, elinewidth=elw, color='k', ecolor='k', zorder=2)

s_CygX1 = ax.scatter(aFe_CygX1[0], aCF_CygX1[0], marker=mark_CygX1, s=symsize, facecolors='w', edgecolors='k', linewidths=elw, zorder=3)
ax.errorbar([aFe_CygX1[0],aFe_CygX1[0]], [aCF_CygX1[0],aCF_CygX1[0]], \
            yerr=[[0.0,0.0], [1.0-aCF_CygX1[0],1.0-aCF_CygX1[0]]], \
            fmt='none', marker=None, markeredgewidth=mew, capsize=csize, elinewidth=elw, color='k', ecolor='k', zorder=2)
ax.errorbar([aFe_CygX1[0],aFe_CygX1[0]], [aCF_CygX1[0],aCF_CygX1[0]], \
            xerr=[[aFe_CygX1[1],aFe_CygX1[1]], [aFe_CygX1[2],aFe_CygX1[2]]], \
            fmt='none', marker=None, markeredgewidth=mew, capsize=csize, elinewidth=elw, color='k', ecolor='k', zorder=2)

s_1915  = ax.scatter(aFe_1915[0], aCF_1915[0], marker=mark_1915, s=symsize, facecolors='w', edgecolors='k', linewidths=elw, zorder=3)
ax.errorbar([aFe_1915[0],aFe_1915[0]], [aCF_1915[0],aCF_1915[0]], \
            yerr=[[0.0,0.0], [1.0-aCF_1915[0],1.0-aCF_1915[0]]], \
            fmt='none', marker=None, markeredgewidth=mew, capsize=csize_arrow, elinewidth=elw, color='k', ecolor='k', zorder=2, ylolims=True)
ax.errorbar([aFe_1915[0],aFe_1915[0]], [aCF_1915[0],aCF_1915[0]], \
            xerr=[[aFe_1915[1],aFe_1915[1]], [aFe_1915[2],aFe_1915[2]]], \
            fmt='none', marker=None, markeredgewidth=mew, capsize=csize, elinewidth=elw, color='k', ecolor='k', zorder=2)

s_J1655 = ax.scatter(aFe_J1655[0], aCF_J1655[0], marker=mark_J1655, s=symsize, facecolors='w', edgecolors='k', linewidths=elw, zorder=3)
ax.errorbar([aFe_J1655[0],aFe_J1655[0]], [aCF_J1655[0],aCF_J1655[0]], \
            yerr=[[aCF_J1655[1],aCF_J1655[1]], [aCF_J1655[2],aCF_J1655[2]]], \
            fmt='none', marker=None, markeredgewidth=mew, capsize=csize, elinewidth=elw, color='k', ecolor='k', zorder=2)
ax.errorbar([aFe_J1655[0],aFe_J1655[0]], [aCF_J1655[0],aCF_J1655[0]], \
            xerr=[[aFe_J1655[1],aFe_J1655[1]], [aFe_J1655[2],aFe_J1655[2]]], \
            fmt='none', marker=None, markeredgewidth=mew, capsize=csize, elinewidth=elw, color='k', ecolor='k', zorder=2)

s_LMCX1 = ax.scatter(aFe_LMCX1[0], aCF_LMCX1[0], marker=mark_LMCX1, s=symsize, facecolors='w', edgecolors='k', linewidths=elw, zorder=3)
ax.errorbar([aFe_LMCX1[0],aFe_LMCX1[0]], [aCF_LMCX1[0],aCF_LMCX1[0]], \
            yerr=[[aCF_LMCX1[1],aCF_LMCX1[1]], [aCF_LMCX1[2],aCF_LMCX1[2]]], \
            fmt='none', marker=None, markeredgewidth=mew, capsize=csize, elinewidth=elw, color='k', ecolor='k', zorder=2)
ax.errorbar([aFe_LMCX1[0],aFe_LMCX1[0]], [aCF_LMCX1[0],aCF_LMCX1[0]], \
            xerr=[[aFe_LMCX1[1],aFe_LMCX1[1]], [aFe_LMCX1[2],aFe_LMCX1[2]]], \
            fmt='none', marker=None, markeredgewidth=mew, capsize=csize, elinewidth=elw, color='k', ecolor='k', zorder=2)

s_J1550 = ax.scatter(aFe_J1550[0], aCF_J1550[0], marker=mark_J1550, s=symsize*2.0, facecolors='w', edgecolors='k', linewidths=elw, zorder=3)
ax.errorbar([aFe_J1550[0],aFe_J1550[0]], [aCF_J1550[0],aCF_J1550[0]], \
            xerr=[[aFe_J1550[1],aFe_J1550[1]], [aFe_J1550[2],aFe_J1550[2]]], \
            yerr=[[aCF_J1550[1],aCF_J1550[1]], [aCF_J1550[2],aCF_J1550[2]]], \
            fmt='none', marker=None, markeredgewidth=mew, capsize=csize, elinewidth=elw, color='k', ecolor='k', zorder=2)

ax.text(aFe_U1543[0], aCF_U1543[0]-0.13, r'$4{\rm U}\ 1543$'+'--'r'$47$', verticalalignment='center', horizontalalignment='center', size=ts)
ax.text(aFe_CygX1[0]-0.06, aCF_CygX1[0], r'${\rm Cyg}$'+'\n'+r'${\rm X}$'+'--'r'$1$', verticalalignment='center', horizontalalignment='center', size=ts)
ax.text(aFe_1915[0]+0.095, aCF_1915[0], r'${\rm GRS}$'+'\n'+r'$1915$', verticalalignment='center', horizontalalignment='right', size=ts)
ax.text(aFe_J1655[0], aCF_J1655[0]-0.13, r'${\rm GRO\ J}1655$'+'--'r'$40$', verticalalignment='center', horizontalalignment='center', size=ts)
ax.text(aFe_LMCX1[0]+0.065, aCF_LMCX1[0]-0.05, r'${\rm LMC}$'+'\n'+r'${\rm X}$'+'--'r'$1$', verticalalignment='center', horizontalalignment='center', size=ts)
ax.text(aFe_J1550[0]+aFe_J1550[2], aCF_J1550[0]-0.05, r'${\rm XTE\ J}1550$'+'--'r'$564$', verticalalignment='center', horizontalalignment='center', size=ts)

fig.savefig(fout, bbox_inches=0, dpi=dpi)
plt.close()


import sys, os, re
import json
from FlokAlgorithmLocal import FlokDataFrame, FlokAlgorithmLocal
from nltk import word_tokenize
from nltk.stem.porter import PorterStemmer

class Batch_TextStemming(FlokAlgorithmLocal):

    def run(self, inputDataSets, params):
        text_dict = inputDataSets.get(0)
        porter_stemmer = PorterStemmer()
        punctuation = "[’。，！…!#$)“%&\(”*+,-.'/:;<=>?@[\\]^_`{|}~+]"
        for text_name,text in text_dict.items():
            process_result = []
            text = re.sub(punctuation, "", text)
            tokens = word_tokenize(text)  # 分词
            for word in tokens:
                stem_word = porter_stemmer.stem(word)
                process_result.append(stem_word)
            text_dict[text_name] = " ".join(process_result)
        result = FlokDataFrame()
        result.addDF(text_dict)
        return result

if __name__ == "__main__":
    all_info = json.loads(sys.argv[1])
    # f = open("test.json", encoding='utf-8')
    # all_info = json.load(f)
    # all_info = {
    #     "input": ["data/english.txt"],
    #     "inputFormat":["txt"],
    #     "inputLocation": ["local_fs"],
    #     "output": ["data/stemming_result.txt"],
    #     "outputFormat": ["txt"],
    #     "outputLocation":["local_fs"],
    #     "parameters": {""}
    # }
    params = all_info["parameters"]
    inputPaths = all_info["input"]
    inputTypes = all_info["inputFormat"]
    inputLocation = all_info["inputLocation"]
    outputPaths = all_info["output"]
    outputTypes = all_info["outputFormat"]
    outputLocation = all_info["outputLocation"]
    algorithm = Batch_TextStemming()

    dataSet = algorithm.read(inputPaths, inputTypes, inputLocation, outputPaths, outputTypes)
    result = algorithm.run(dataSet, params)
    algorithm.write(outputPaths, result, outputTypes, outputLocation)


#!/usr/bin/env python

import argparse
import os
import zipfile
import gzip
import tarfile
import contextlib
import tempfile
import json
import subprocess
import pathlib
import pandas as pd
from glob import glob
import multiprocessing as mp
import shutil
import re
import sys

ANALYSIS_NOTEBOOK_TEMPLATE = 'Analysis-Template.ipynb'

def mkdirp(dirpath):
    pathlib.Path(dirpath).mkdir(parents=True, exist_ok=True)

def parse_args():
    parser = argparse.ArgumentParser()
    commands = parser.add_subparsers()
    extract_cmd = commands.add_parser('extract', help='extract test outputs from testground output archive')
    extract_cmd.add_argument('test_output_zip_path', nargs=1,
                             help='path to testground output zip or tgz file')

    extract_cmd.add_argument('--output-dir', '-o', dest='output_dir', default=None,
                             help='path to write output files. default is to create an "analysis" dir next to archive file')
    extract_cmd.set_defaults(subcommand='extract')

    run_notebook_cmd = commands.add_parser('run_notebook',
                                           help='runs latest analysis notebook against extracted test data')
    run_notebook_cmd.add_argument('--mesh', action='store_true', default=False,
                                  help='if true, derive the mesh state when converting data. This takes a long time, so the default is false.')
    run_notebook_cmd.add_argument('test_result_dir', nargs='+',
                                  help='directories to run against. must contain an "analysis" subdir with extracted test data')
    run_notebook_cmd.set_defaults(subcommand='run_notebook')
    return parser.parse_args()

def concat_files(names, outfile):
    for name in names:
        with open(name, 'rb') as f:
            outfile.write(f.read())

# depending on which test runner was used, the collection archive may be either a zip (local docker & exec runner),
# or a tar.gz file (k8s). Unfortunately, the zipfile and tarfile modules are different species of waterfowl,
# so we duck typing doesn't help. So this method extracts whichever one we have to a temp directory and
# returns the path to the temp dir.
# use as a context manager, so the temp dir gets deleted when we're done:
#   with open_archive(archive_path) as a:
#     files = glob(a + '/**/tracer-output')
@contextlib.contextmanager
def open_archive(archive_path):
    # zipfile and tarfile both have an extractall method, at least
    if zipfile.is_zipfile(archive_path):
        z = zipfile.ZipFile(archive_path)
    else:
        z = tarfile.open(archive_path, 'r:gz')

    with tempfile.TemporaryDirectory(prefix='pubsub-tg-archive-') as d:
        z.extractall(path=d)
        yield d

# sugar around recursive glob search
def find_files(dirname, filename_glob):
    path = '{}/**/{}'.format(dirname, filename_glob)
    return glob(path, recursive=True)

PEER_INFO_PATTERN = re.compile(
    r'Host peer ID: ([0-9a-zA-Z]+), seq (\d+), node type: ([a-z]+), node type seq: (\d+), node index: (\d+) / (\d+)')

def extract_peer_info(run_out):
    infos = []
    with open(run_out, 'rt') as f:
        for line in f.readlines():
            m = PEER_INFO_PATTERN.search(line)
            if m:
                pid = m.group(1)
                seq = int(m.group(2))
                node_type = m.group(3)
                node_type_seq = int(m.group(4))
                node_index = int(m.group(5))
                node_index_bound = int(m.group(6))
                infos.append(
                    {'peer_id': pid,
                     'type': node_type,
                     'seq': seq,
                     'node_type_seq': node_type_seq,
                     'node_index': node_index,
                     'node_index_bound': node_index_bound})
    if len(infos) == 0:
        print('warning: no peer info found in {}'.format(run_out))
    return infos

def extract_timing_info(run_out, node_type):
    if node_type == 'honest':
        times = dict(t_warm=0, t_connect=0, t_run=0, t_cool=0, t_complete=0)
    else:
        times = dict(t_connect=0)

    with open(run_out, 'rt') as f:
        for line in f.readlines():
            try:
                obj = json.loads(line)
            except BaseException as err:
                print("error parsing run output: ", err)
                continue
            if 'ts' not in obj or 'event' not in obj or obj['event'].get('type', '') != 'message':
                continue
            msg = obj['event']['message']
            ts = obj['ts']
            if re.match(r'connecting to peers.*', msg):
                times['t_connect'] = ts
                continue

            # the rest of the times are only logged by honest peers
            if node_type != 'honest':
                continue
            if re.match(r'Wait for .* warmup time', msg):
                times['t_warm'] = ts
                continue
            if re.match(r'Wait for .* run time', msg):
                times['t_run'] = ts
                continue
            if re.match(r'Run time complete, cooling down.*', msg):
                times['t_cool'] = ts
                continue
            if msg == 'Cool down complete':
                times['t_complete'] = ts
                continue

    for k, v in times.items():
        if v == 0:
            print('warning: unable to determine time value for {}'.format(k))
    return times

def extract_peer_and_timing_info(run_out_files):
    entries = []
    for filename in run_out_files:
        infos = extract_peer_info(filename)
        if infos is None or len(infos) == 0:
            continue

        # FIXME: we only get the timing info for the first peer in the container
        # unfortunately, getting the times for individual peers requires code changes
        times = extract_timing_info(filename, infos[0].get('type', 'unknown'))
        for info in infos:
            info.update(times)
            entries.append(info)
    return entries

def aggregate_output(output_zip_path, out_dir):
    topology = dict()

    with open_archive(output_zip_path) as archive:
        tracefiles = find_files(archive, 'tracer-output*')
        names = [f for f in tracefiles if 'full' in f]
        if len(names) > 0:
            with gzip.open(os.path.join(out_dir, 'full-trace.bin.gz'), 'wb') as gz:
                concat_files(names, gz)

        names = [f for f in tracefiles if 'filtered' in f]
        if len(names) > 0:
            with gzip.open(os.path.join(out_dir, 'filtered-trace.bin.gz'), 'wb') as gz:
                concat_files(names, gz)

        # copy aggregate metrics files
        names = [f for f in tracefiles if 'aggregate' in f]
        for name in names:
            dest = os.path.join(out_dir, os.path.basename(name))
            shutil.copyfile(name, dest)

        # copy peer score files
        names = find_files(archive, 'peer-scores*')
        for name in names:
            dest = os.path.join(out_dir, os.path.basename(name))
            shutil.copyfile(name, dest)

        # get peer id -> seq mapping & timing info from run.out files
        names = find_files(archive, 'run.out')
        info = extract_peer_and_timing_info(names)
        dest = os.path.join(out_dir, 'peer-info.json')
        with open(dest, 'wt') as f:
            json.dump(info, f)

        # Collect contents of all files of the form 'connections-honest-8-1'
        names = find_files(archive, 'connections*')
        for name in names:
            with open(name, 'r') as infile:
                name = os.path.basename(name)
                _, node_type, node_type_seq, node_idx = name.split('.')[0].split('-')
                conns = json.loads(infile.read())
                topology[node_type + '-' + node_type_seq + '-' + node_idx] = conns or []

    # Write out topology file
    top_path = os.path.join(out_dir, 'topology.json')
    with open(top_path, 'wt') as outfile:
        outfile.write(json.dumps(topology))

def run_tracestat(tracer_output_dir):
    full = os.path.join(tracer_output_dir, 'full-trace.bin.gz')
    filtered = os.path.join(tracer_output_dir, 'filtered-trace.bin.gz')
    if os.path.exists(full):
        tracer_output = full
    elif os.path.exists(filtered):
        tracer_output = filtered
    else:
        print('no event tracer output found, skipping tracestat')
        return

    print('running tracestat on {}'.format(tracer_output))
    try:
        cmd = ['go', 'run', 'github.com/libp2p/go-libp2p-pubsub-tracer/cmd/tracestat', '-cdf', tracer_output]
        p = subprocess.run(cmd, capture_output=True, text=True, check=True)
    except BaseException as err:
        print('error calling tracestat: ', err)
        return

    # split output into summary and latency CDF
    [summary, cdf] = p.stdout.split('=== Propagation Delay CDF (ms) ===')

    with open(os.path.join(tracer_output_dir, 'tracestat-summary.txt'), 'w', encoding='utf8') as f:
        f.write(summary)
    with open(os.path.join(tracer_output_dir, 'tracestat-cdf.txt'), 'w', encoding='utf8') as f:
        f.write(cdf)

    print(summary)

def extract_test_outputs(test_output_zip_path, output_dir=None, convert_to_pandas=False, prep_notebook=True):
    if output_dir is None or output_dir == '':
        output_dir = os.path.join(os.path.dirname(test_output_zip_path), 'analysis')

    raw_output_dir = os.path.join(output_dir, 'raw-data')
    mkdirp(raw_output_dir)
    aggregate_output(test_output_zip_path, raw_output_dir)
    run_tracestat(raw_output_dir)

    if convert_to_pandas:
        import notebook_helper
        print('converting data to pandas format...')
        pandas_dir = os.path.join(output_dir, 'pandas')
        notebook_helper.to_pandas(raw_output_dir, pandas_dir)
    if prep_notebook:
        prepare_analysis_notebook(analysis_dir=output_dir)
    return output_dir

def prepare_analysis_notebook(analysis_dir):
    notebook_out = os.path.join(analysis_dir, 'Analysis.ipynb')
    shutil.copy(ANALYSIS_NOTEBOOK_TEMPLATE, notebook_out)
    shutil.copy('./notebook_helper.py', os.path.join(analysis_dir, 'notebook_helper.py'))
    print('saved analysis notebook to {}'.format(notebook_out))

def run_analysis_notebook(analysis_dir, derive_meshes=False):
    prepare_analysis_notebook(analysis_dir)
    notebook_path = os.path.join(analysis_dir, 'Analysis.ipynb')
    cmd = ['papermill', ANALYSIS_NOTEBOOK_TEMPLATE, notebook_path, '--cwd', analysis_dir]
    if derive_meshes:
        cmd += ['-p', 'DERIVE_MESHES', 'True']
    try:
        subprocess.run(cmd, check=True)
    except BaseException as err:
        print('error executing notebook: {}'.format(err), file=sys.stderr)
        return

def run_notebooks(test_result_dirs, derive_meshes=False):
    for d in test_result_dirs:
        analysis_dir = os.path.join(d, 'analysis')
        if not os.path.exists(analysis_dir):
            print('no analysis dir at {}, ignoring'.format(analysis_dir), file=sys.stderr)
            continue
        print('running analysis in {}'.format(analysis_dir))
        run_analysis_notebook(analysis_dir, derive_meshes=derive_meshes)

def run():
    args = parse_args()
    if args.subcommand == 'extract':
        zip_filename = args.test_output_zip_path[0]
        extract_test_outputs(zip_filename, args.output_dir)
    elif args.subcommand == 'run_notebook':
        run_notebooks(args.test_result_dir, derive_meshes=args.mesh)
    else:
        print('unknown subcommand', file=sys.stderr)
        sys.exit(1)

if __name__ == '__main__':
    run()


import os
import threading
import telebot
from telebot import types
import subprocess
import style_transfer as st_t
import time

def time_to_send(user): #отслеживание появления файла из внешней нейросети
    timer[user] = time.time()
    while True:
        if (time.time() - timer[user]) >= 15:
            if os.path.exists(str(user)+"image3.jpg"):
                with open(str(user)+"image3.jpg", 'rb') as test_image:
                    bot.send_photo(user, test_image, caption='Представляю Вам своё творение. Понадобились недюжинные усилия, но я справился')
                    question_one(user)
                    break
            else:
                timer[user] = time.time()

def delete_user(user, n): #удаление отработанных файлов
    if n == 1: #удаляем начиная заново
        if user in im_name_1:
            im_name_1.pop(user)
    if (n == 1) or (n == 2): #удаляем начиная заново или при смене стиля
        if user in im_name_2:
            im_name_2.pop(user)
        if user in im_name_3:
            im_name_3.pop(user)

def delete_photo(user, n): #удаление отработанных файлов
    if n == 1: #удаляем начиная заново
        if os.path.exists(str(user)+"image1.jpg"):
            os.remove(str(user)+"image1.jpg")
    if (n == 1) or (n == 2): #удаляем начиная заново или при смене стиля
        if os.path.exists(str(user)+"image2.jpg"):
            os.remove(str(user)+"image2.jpg")
        if os.path.exists(str(user)+"image3.jpg"):
            os.remove(str(user)+"image3.jpg")

bot = telebot.TeleBot('token') #адрес бота
im_name_1 = {} #картинка для обработки
im_name_2 = {} #картинки со стилем
im_name_3 = {} #результат
timer = {} #таймер

def help(message):
    user_id = message.chat.id
    bot.send_message(user_id, 'Приветствую! Я талантлвый художник в изгнании.\
    Я отказался от известности, но тоскую по своему ремеслу, и не хочу, чтобы мой прекрасный талант прозябал\
    в безделии и унынии.')
    bot.send_message(user_id, 'Да, я подделывал работы других мастеров, известнейших художников.\
    Но с каким упоением и ни с чем несравнимой радостью я это делал!')
    bot.send_message(user_id, 'Вот пришлите мне совершенно любую картинку, которую хотите изменить.\
    А вслед за нею картину того художника, чей стиль я должен подделать. Уверяю, вы будете удивлены.')

@bot.message_handler(func=lambda message: True, content_types=['text'])
def send_text(message):
    user_id = message.chat.id
    if ('привет' or 'здравствуй') in message.text.lower():
        bot.send_message(user_id, 'Приветствую, пришлите мне картину, которую вы хотите изменить')
    elif ('пока') in message.text.lower():
        bot.send_message(user_id, 'Наши встречи очень много для меня значат. Прощайте.....')
        delete_photo(user_id, 1)
        delete_user(user_id, 1)
        #bot.stop_polling()
    elif ('сотри' or 'удали') in message.text.lower():
        delete_photo(user_id, 1)
        delete_user(user_id, 1)
        bot.send_message(user_id, 'Все следы уничтожены')
    elif ('как дела'or 'как ты') in message.text.lower():
        bot.send_message(user_id, 'Кто-то может сказать, что я счастливчик, но  тем не менее, всё могло быть куда лучше...')
    elif ('вопрос' or 'проблема') in message.text.lower():
        with open('QA.txt', 'a') as bot_feedback:
            bot_feedback.write(str(user_id) + ' ' + message.text + '\n')
        bot.send_message(user_id, 'Отлично, я положил вашу записку в странное приспособление, оставленное  высшими силами.\
        Кто знает, что может произойти?')
    else:
        bot.send_message(user_id, 'Всё-таки, я достаточно занятой человек, чтобы отвечать на вопросы не по моей работе.\
        Но если написать в сообщении слово "вопрос", то силы, что несоизмеримо выше меня, услышат вас\
        А ещё можно написать /help')

@bot.message_handler(func=lambda message: True, content_types=['photo'])
def photo(message):
    user_id = message.chat.id
    fileID = message.photo[-1].file_id
    #если пользователь ещё не отсылал картинок, записываем адрес первой картинки
    if user_id not in im_name_1:
        im_name_1[user_id] = message.photo[-1].file_id
        bot.send_message(message.chat.id, 'У вас отличный вкус!\
        А сейчас пришлите мне картину того художника, чей стиль я должен подделать')
    else:
    	#если пользователь что-то отослал, записываем адрес второй картинки
        im_name_2[user_id] = message.photo[-1].file_id
        #хендлер для выбора способа обработки
        chose_one(user_id)

@bot.callback_query_handler(func=lambda call: True)
def user_answer(call):
    user = call.message.chat.id
    if ((call.data == "0") or (call.data =="1")):
        delete_or_no(user, call.data)
    else:
        convert_image(user, call.data)

def question_one(user):
    if user in im_name_3:
        keyboard = types.InlineKeyboardMarkup()
        keyboard.add(types.InlineKeyboardButton(text="Изменить", callback_data= "0"))
        keyboard.add(types.InlineKeyboardButton(text="Следующая картина", callback_data="1"))
        bot.send_message(user, 'Хотите изменить стилистику, либо изменить следующую картину?', reply_markup=keyboard)

def chose_one(user):
    if user not in im_name_3:
        keyboard = types.InlineKeyboardMarkup()
        keyboard.add(types.InlineKeyboardButton(text="Хочу быстрее", callback_data= "fst"))
        keyboard.add(types.InlineKeyboardButton(text="Могу и подождать", callback_data="slw"))
        bot.send_message(user, 'Хотите результат побыстрее, либо дадите время на детальную проработку?', reply_markup=keyboard)

def convert_image(user, answer):
    if answer == "fst": #быстрый алгоритм
        if len(im_name_1[user]) <= 150: #при повторном запуске в ячейке может быть картинка, а не id файла
            im_name_1[user] = bot.download_file(bot.get_file(im_name_1[user]).file_path)
        im_name_2[user] = bot.download_file(bot.get_file(im_name_2[user]).file_path)
        #запускаем через thread для обеспецения асинхронности
        threading.Thread(target=st_t.photo_connect(im_name_1[user], im_name_2[user], user, im_name_3)).start()
        bot.send_photo(user, im_name_3[user]\
                           , caption='Представляю Вам своё творение. Понадобились недюжинные усилия, но я справился')
        question_one(user)
    elif answer == "slw": #медленный но каественный алгоритм
        with open(str(user)+"image1.jpg", 'wb') as new_file:
            if len(im_name_1[user]) <= 150:
                #при повторном запуске в ячейке может быть картинка, а не id файла
                new_file.write(bot.download_file(bot.get_file(im_name_1[user]).file_path))
            else:
                new_file.write(im_name_1[user])
        with open(str(user)+"image2.jpg", 'wb') as new_file:
            new_file.write(bot.download_file(bot.get_file(im_name_2[user]).file_path))
        im_name_3[user] = str(user)+"image3.jpg"
        bot.send_message(user, 'Тогда я погружаюсь в творческий процесс, на несколько  минут, обычно от двух до пяти.')
        #команда для запуска нейросети. Для снижения нагрузки на память использует cudnn, оптимизатор lbfgs\
        #выдаёт наилучший результат по качеству, хоть и проигрывает adam-у в скорости (порядка 60 секунд)
        command = "python neural_style.py -style_image {} -content_image {} -output_image {}\
         -backend cudnn -optimizer lbfgs".format(str(user)+"image2.jpg", str(user)+"image1.jpg", im_name_3[user])
        p = subprocess.Popen(command) #запускаем сторонний процесс, может одновременно выполняться для нескольких пользователей
        threading.Thread(target=time_to_send(user)).start() #отслеживаем появление результата в папке

def delete_or_no(user, answer):
    if answer == "1":
        delete_photo(user, 1) #удаляем все файлы
        delete_user(user, 1) #очищаем память
        bot.send_message(user, 'Пришлите мне картину, которую вы хотите изменить')
    elif answer == "0":
        delete_user(user, 2) #удаляем всё кроме первого файла
        delete_photo(user, 2) #очищаем память кроме первого файла
        bot.send_message(user, 'Пришлите мне иную картину для вдохновения')

#т.к. возникли проблемы с запуском webhook на локальной машине,
# а времени на веб-сервер из-за загруженности не хватило, обход ошибок телеги выполняем при помощи try/except
while True:
    try:
        bot.polling(none_stop=True, interval=0)
    except:
        time.sleep(5)
"""neural_networks:
    fast: TensorFlow tutorial https://www.tensorflow.org/tutorials/generative/style_transfer
    qualitative: Johnson, Justin, https://github.com/jcjohnson/neural-style
    bot: c-stone studio, Gleb Yakimovich, https://github.com/kolovrado"""


import os

from pydantic import BaseSettings

class Settings(BaseSettings):
    FASTAPI_APP_ALLOW_ORIGINS: str
    REDIS_HOST: str
    REDIS_PORT: int

    class Config:
        env_file = os.getenv('ENV_FILE')

settings = Settings()


"""
Remove RasterAggregatedLayer and Related objects given RasterAggregatedLayer ids.
Note: Use the 'list_raster_layers' command to obtain the RasterAggregatedLayer ids.
"""
from django.core.management.base import BaseCommand
from django.conf import settings
from ...models import RasterAggregatedLayer

WGS84_SRID = settings.WGS84_SRID

class Command(BaseCommand):
    help = __doc__

    def add_arguments(self, parser):
        parser.add_argument("-i", "--ids",
                            type=int,
                            default=None,
                            required=True,
                            nargs="+",
                            help="RasterAggregatedLayer ids to delete.")

    def handle(self, *args, **options):
        for layer in RasterAggregatedLayer.objects.order_by("id"):
            self.stdout.write("Removing RasterAggregatedLayer: [{}] {}...".format(layer.id, layer.name))
            # Delete related DataModel Objects
            layer.pixels().delete()

            # delete Layer
            layer.delete()

            self.stdout.write("Done!")


import os
import torch
from cross_domain_predictor import GCN_predictor
import argparse
from scipy.stats import kendalltau
import numpy as np
from torch.utils.data import DataLoader
from dataset_matrix import Dataset_Darts
import matplotlib.pyplot as plt
import pickle
from torch import optim
from utils import get_logger, to_cuda, convert_to_genotype
from torch import nn

filename = os.path.join('tiny_darts', 'darts_dataset.pth.tar')
train_data = torch.load(os.path.join('path', 'darts_dataset_fullytrain.pth.tar'))
data = torch.load(filename)

k=50
train_data['dataset'] = train_data['dataset'][:k]
train_data['best_acc_list'] = train_data['best_acc_list'][:k]

# train_data['dataset'] = data['dataset'][:k]
# train_data['best_acc_list'] = data['best_acc_list'][:k]

# data['dataset'] = data['dataset'][k:]
# data['best_acc_list'] = data['best_acc_list'][k:]

# # 1,2 <-> 3,4 op
# for geno in train_data['dataset']:
#     for i, g in enumerate(geno):
#         for j, op in enumerate(g):
#             pre= op[0]
#             if op[1] ==1:
#                 geno[i][j] = (pre, 3)
#             if op[1] ==2:
#                 geno[i][j] = (pre, 4)
#             if op[1] ==3:
#                 geno[i][j] = (pre, 1)
#             if op[1] ==4:
                # geno[i][j] = (pre, 2)

parser = argparse.ArgumentParser(description='darts_test')
parser.add_argument('--integers2one_hot', type=bool, default=True, help='whether to transform integers -> one_hot')
parser.add_argument('--train_batch_size', default=100, type=int)
parser.add_argument('--test_batch_size', default=100, type=int)
parser.add_argument('--seed', type=int, default=2, help='random seed')
parser.add_argument('--gpu', type=int, default=0, help='gpu device id')
parser.add_argument('--which_assistant', type=int, default=0, help='0 for tiny, 1 for small tiny')
parser.add_argument('--ns', default=True, type=bool, help='whether to forbidden skip in assistant space')
parser.add_argument('--show_figure', default=False, type=bool)
parser.add_argument('--figure_index', default=2, type=int, help='the index of the saving figure')
# parser.add_argument('--predictor', type=str, default='GCN', choices=['RF', 'GCN'])
args = parser.parse_args()

if __name__ == '__main__':
    normal_layers = 6

    print("args =", args)

    torch.cuda.set_device(args.gpu)
    np.random.seed(args.seed)
    torch.manual_seed(args.seed)
    torch.cuda.manual_seed(args.seed)

    Darts_Matrix = Dataset_Darts(dataset_num=len(train_data['dataset']), dataset=train_data['dataset'])
    dataloader_darts = DataLoader(Darts_Matrix, batch_size=args.test_batch_size, shuffle=False)

    predictor = GCN_predictor(None)
    predictor.train_without_domain_adaptation(dataloader_darts, train_data['best_acc_list'])

    Darts_Matrix = Dataset_Darts(dataset_num=len(data['dataset']), dataset=data['dataset'])
    dataloader_darts = DataLoader(Darts_Matrix, batch_size=args.test_batch_size, shuffle=False)
    pred_y = predictor.predict(dataloader_darts, normal_layer=normal_layers)

    true_y = np.array(data['best_acc_list'])

    print('KTau: {}'.format(kendalltau(pred_y, true_y)[0]))


import dask
import dask.dataframe as dd
import pandas as pd
import s3fs
import json
import dask.bag as db
import ast
import logging
import os
import datetime

import logging_utils as logUtils

def get_current_datetime():
    return str(datetime.datetime.now().strftime("%Y%m%d_%H%M%S"))

def read_csv_from_url(data_url, data_file_cols=None):
    rating_movie_tv = dd.read_csv(data_url)
    rating_movie_tv.columns = data_file_cols
    return rating_movie_tv

def read_json_from_url(meta_url):
    meta_movie_tv = db.read_text(meta_url).map(ast.literal_eval)
    return meta_movie_tv

def write_datafile_to_S3(ddf, path, st_opt):
    ddf.to_csv(path, storage_options=st_opt)

def write_metadatafile_to_S3(dbag, path, st_opt):
    dbag.map(json.dumps).to_textfiles(path, storage_options=st_opt)

if __name__ == "__main__":
    logger = logging.getLogger("ingestion")
    err_code = 0
    app_name = "Data_Loading_toS3"

    code_home = os.getcwd()
    log_home = os.path.join(code_home, "log")

    if not os.path.exists(log_home):
        os.mkdir(log_home)

    log_name = "{0}_{1}.log".format(get_current_datetime(), app_name)

    log_path = os.path.join(log_home, log_name)
    logUtils.init_log(log_path)

    try:
        logger.info("-----------------------------------Script Commencement------------------------------------------")
        logger.info("Arguements Passed : {0}".format(args))

        config_file_path = os.path.join(code_home, "config.json")
        f = open(config_file_path)
        config = json.load(f)

        logger.info("Reading Data file from URL")
        rating_mv_tv_data = read_csv_from_url(config["DataFileUrl"], config["DataFileColumns"])

        logger.info("Writing Data file to S3")
        str_options = {"anon" : False, "key" : config["S3AccessKey"], "secret" : config["S3SecretKey"]}
        s3_file_path = "/".join["s3:/", config["S3BucketName"], config["S3DataFileName"]]
        logger.info("s3_file_path : ", s3_file_path)
        write_datafile_to_S3(rating_mv_tv_data, s3_file_path, str_options)

        logger.info("Reading MetaData file from URL")
        rating_mv_tv_meta = read_json_from_url(config["MetadataFileUrl"])

        logger.info("Writing MetaData file to S3")
        s3_meta_file_path = "/".join["s3:/", config["S3BucketName"], config["S3MetadataFileName"]]
        logger.info("s3_meta_file_path : ", s3_meta_file_path)
        write_metadatafile_to_S3(rating_mv_tv_meta, s3_meta_file_path, str_options)

    except Exception as e:
        logger.error(e)
        err_code = 1

    finally:
        logger.info("Progress exit with error code : {0}".format(err_code))
        exit(err_code)


#!/usr/bin/env python
#-*- coding: iso-8859-1 -*-
###############################################################################
#
# This module contains implementation of an RPC interface and resource, built
# on top of TcpInterface/TcpResource. Pythomnic3k RPC is based on the following
# principles:
#
# 1. Automatic discovery of a target cage. Whenever some cage executes
#    pmnc("othercage").module.method(...)
#    the location of othercage is discovered automatically and transparently
#    by sending out UDP broadcasts to which (either copy of) othercage presumably
#    responds. The received response contains the listening address of the
#    responded cage and TCP/SSL connection can be initiated.
#    Target cage addresses can be configured separately, if necessary,
#    for example if UDP broadcasts can't go freely between the cages.
#    Therefore a network of cages can well exist without broadcasts,
#    but that would require a lot of manual configuration.
#
# 2. Multiple cages with the same name can be running on different servers,
#    they all respond to the discovery requests. One of the responses to arrive
#    first is chosen, and responses from different cages may cause connections
#    to different copies of the target cage to exist at the same time.
#
# 3. Hence this - in Pythomnic3k there is no difference between copies of
#    identically named cages - they are supposed to be freely interchangeable.
#    This fact is stressed by using SSL certificates to authenticate cages
#    to each other - each "cage" presumably owns a certificate which has common
#    name of "cage". This allows cages to trust each other at least by name.
#
# 4. RPC implemented here is synchronous and plays by Pythomnic3k rules of non-
#    blocking and timing out within deadline. Execution of the call on the target
#    cage inherits the original timeout, therefore the original request is logically
#    "extended" to the target cage and impersonated for the course of RPC execution.
#
# 5. Each cage advertises its location by periodically sending out broadast
#    requests. The received locations of other cages are kept on record and
#    can be accessed by get_cages and get_nodes methods of the rpc interface,
#    this is used by the health_monitor cage to locate active cages. Typically
#    a cage broadcasts its own presence but it is technically possible for one
#    cage (a sort of registry perhaps) to broadcast the locations of other cages.
#
# There is just one RPC interface and RPC resource required for a cage to be
# able to exchange RPC calls with other cages.
#
# Sample RPC interface configuration (config_interface_rpc.py):
#
# config = dict \
# (
# protocol = "rpc",                                        # meta
# request_timeout = None,                                  # meta, optional
# random_port = -63000,                                    # tcp, negative means "in range 63000..63999"
# max_connections = 100,                                   # tcp
# broadcast_address = ("1.2.3.4/1.2.3.255", 12480),        # rpc, "interface address/broadcast address", port
# flock_id = "DEFAULT",                                    # rpc, arbitrary cage group identifier
# )
#
# Sample RPC resource configuration (config_resource_rpc.py)
#
# config = dict \
# (
# protocol = "rpc",                                        # meta
# broadcast_address = ("1.2.3.4/1.2.3.255", 12480),        # rpc, "interface address/broadcast address", port
# discovery_timeout = 3.0,                                 # rpc + tcp (discovery + connect timeout)
# multiple_timeout_allowance = 0.5,                        # rpc, in range 0.0..1.0
# flock_id = "DEFAULT",                                    # rpc, arbitrary cage group identifier
# exact_locations = { "SomeCage": "ssl://1.2.3.5:63842" }, # rpc, maps cage names to their fixed locations
# )

# Pythomnic3k project
# (c) 2005-2010, Dmitry Dvoinikov <dmitry@targeted.org>
# Distributed under BSD license
#
###############################################################################

__all__ = [ "Interface", "Resource", "Handler" ]

###############################################################################

import os; from os import urandom, SEEK_SET, SEEK_CUR, SEEK_END, path as os_path
import time; from time import time
import binascii; from binascii import b2a_hex
import select; from select import select
import io; from io import BytesIO
import hashlib; from hashlib import sha1
import threading; from threading import current_thread, Lock
import pickle; from pickle import load as unpickle, dumps as pickles
import random; from random import randint
import ssl; from ssl import CERT_REQUIRED
import socket; from socket import socket, AF_INET, SOCK_DGRAM, SOL_SOCKET, \
                                  SO_BROADCAST, SO_REUSEADDR, error as socket_error
try:
    from socket import SO_REUSEPORT
except ImportError:
    have_reuse_port = False
else:
    have_reuse_port = True

if __name__ == "__main__": # add pythomnic/lib to sys.path
    import os; import sys
    main_module_dir = os.path.dirname(sys.modules["__main__"].__file__) or os.getcwd()
    sys.path.insert(0, os.path.normpath(os.path.join(main_module_dir, "..", "..", "lib")))

import exc_string; from exc_string import exc_string
import typecheck; from typecheck import typecheck, optional, by_regex, dict_of
import pmnc.resource_pool; from pmnc.resource_pool import TransactionalResource, \
                                                          ResourceError, RPCError
import pmnc.timeout; from pmnc.timeout import Timeout
import pmnc.threads; from pmnc.threads import HeavyThread

###############################################################################

valid_cage_name = by_regex("^[A-Za-z0-9_-]{1,32}$")
valid_node_name = by_regex("^[A-Za-z0-9_-]{1,32}$")
valid_flock_id = by_regex("^[A-Za-z0-9_-]+$")
valid_location = by_regex("^(ssl|tcp)://([0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}|\\*):[0-9]{1,5}/$")
valid_exact_location = by_regex("^(ssl|tcp)://[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}:[0-9]{1,5}/$")

###############################################################################

def _get_peer_name(ssl_socket):
    for f in ssl_socket.getpeercert()["subject"]:
        if f[0][0] == "commonName":
            return f[0][1]
    else:
        return None

###############################################################################

def _create_sending_broadcast_socket(addr: str) -> socket:
    bc_socket = socket(AF_INET, SOCK_DGRAM)
    try:
        bc_socket.setsockopt(SOL_SOCKET, SO_BROADCAST, 1)
        bc_socket.bind((addr, 0))
    except:
        bc_socket.close()
        raise
    else:
        return bc_socket

###############################################################################

def _create_receiving_broadcast_socket(port: int) -> socket:
    bc_socket = socket(AF_INET, SOCK_DGRAM)
    try:
        bc_socket.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
        if have_reuse_port:
            bc_socket.setsockopt(SOL_SOCKET, SO_REUSEPORT, 1)
        bc_socket.setsockopt(SOL_SOCKET, SO_BROADCAST, 1)
        bc_socket.bind(("0.0.0.0", port))
    except:
        bc_socket.close()
        raise
    else:
        return bc_socket

###############################################################################

# key files required to establish SSL connection
# may reside in current cage or in .shared

def _locate_key_file(filename: str) -> optional(os_path.isfile):

    s = os_path.join(__cage_dir__, "ssl_keys", filename)
    if os_path.isfile(s):
        return s

    s = os_path.normpath(os_path.join(__cage_dir__, "..", ".shared", "ssl_keys", filename))
    if os_path.isfile(s):
        return s

    return None

###############################################################################

class RpcPacket():

    def __init__(self, max_size: optional(int) = None):
        self._max_size = max_size
        self._stream = BytesIO()
        self._length = None
        self._expected_hash = None
        self._hash = None
        self._method = None

    @staticmethod
    def marshal(data):
        return b"PKL3" + pickles(data)

    def _unmarshal(self):
        if self._method == "PKL3":
            self._stream.seek(52, SEEK_SET)
            return unpickle(self._stream)
        else:
            raise Exception("unsupported marshaling method: "
                            "{0:s}".format(self._method))

    def write(self, data: bytes):

        self._stream.write(data)
        if self._max_size and self._stream.tell() > self._max_size:
            raise Exception("packet size exceeded")

        if self._length is None and self._stream.tell() >= 8:
            self._stream.seek(0, SEEK_SET)
            self._length = int(self._stream.read(8).decode("ascii"), 16) + 48
            if self._max_size and self._length > self._max_size:
                raise Exception("request size exceeded")
            self._stream.seek(0, SEEK_END)

        if self._expected_hash is None and self._stream.tell() >= 48:
            self._stream.seek(8, SEEK_SET)
            self._expected_hash = self._stream.read(40).decode("ascii")
            self._hash = sha1(self._stream.read())
        elif self._hash is not None:
            self._hash.update(data)

        if self._method is None and self._stream.tell() >= 52:
            self._stream.seek(48, SEEK_SET)
            self._method = self._stream.read(4).decode("ascii")
            self._stream.seek(0, SEEK_END)

        if self._stream.tell() == self._length:
            if self._hash.hexdigest().upper() != self._expected_hash:
                raise Exception("request hash mismatch")
            return self._unmarshal()
        else:
            return None

###############################################################################

class Interface: # RPC interface

    @typecheck
    def __init__(self, name: str, *,
                 random_port: lambda i: isinstance(i, int) and (-65500 <= i <= 65535),
                 max_connections: int,
                 broadcast_address: (str, int),
                 flock_id: valid_flock_id,
                 separate_thread_pool: optional(bool) = False,
                 disable_broadcast: optional(bool) = False,
                 request_timeout: optional(float) = None,
                 **kwargs):

        self._name = name
        broadcast_address, self._broadcast_port = broadcast_address
        self._bind_address, self._broadcast_address = broadcast_address.split("/")
        self._disable_broadcast = disable_broadcast
        self._request_prefix = "PYTHOMNIC3K-REQUEST:{0:s}:".format(flock_id)
        self._response_prefix = "PYTHOMNIC3K-RESPONSE:{0:s}:".format(flock_id)
        self._known_cages = {} # { cage: { node: { location: ..., expires_at: ... } } }
        self._known_cages_lock = Lock()
        self._ad_period_idx = 0

        ssl_key_cert_file = _locate_key_file("key_cert.pem")
        ssl_ca_cert_file = _locate_key_file("ca_cert.pem")

        if pmnc.request.self_test == __name__: # self-test
            self.process_rpc_request = kwargs["process_rpc_request"]
            self._cage_name = kwargs["cage_name"]
            if "ad_periods" in kwargs:
                self._ad_periods = kwargs["ad_periods"]
        else:
            self._cage_name = __cage__

        # having handler factory create handlers through a pmnc call
        # allows online modifications to this module, after it is reloaded

        handler_factory = lambda prev_handler: \
            pmnc.protocol_rpc.Handler(self.process_rpc_request, self._cage_name)

        # create an instance of underlying TCP interface, note that in case
        # of SSL we require the client (source cage) to present a certificate

        request_timeout = request_timeout or \
                          pmnc.config_interfaces.get("request_timeout") # this is now static

        self._tcp_interface = \
            pmnc.protocol_tcp.TcpInterface(name, handler_factory, request_timeout,
                                           listener_address = (self._bind_address, random_port),
                                           max_connections = max_connections,
                                           ssl_key_cert_file = ssl_key_cert_file,
                                           ssl_ca_cert_file = ssl_ca_cert_file,
                                           required_auth_level = CERT_REQUIRED)

        # RPC interface is special in that it can be configured to enqueue
        # its requests to a separate private thread pool instead of using
        # the thread pool shared with all other interfaces

        if separate_thread_pool:
            self._thread_pool = pmnc.shared_pools.get_private_thread_pool()
            self._tcp_interface._enqueue_request = self._enqueue_request

    name = property(lambda self: self._tcp_interface.name)
    listener_address = property(lambda self: self._tcp_interface.listener_address)

    ###################################

    def start(self):
        self._tcp_interface.start()
        if not self._disable_broadcast: # create and start broadcaster thread
            self._broadcaster = HeavyThread(target = self._broadcaster_proc,
                                            name = "{0:s}:brc".format(self._name))
            self._broadcaster.start()

    def cease(self):
        if not self._disable_broadcast:
            self._broadcaster.stop()
        self._tcp_interface.cease()

    def stop(self):
        self._tcp_interface.stop()

    ###################################

    def _broadcaster_proc(self):
        try:

            # the listening socket must be bound to the configured port

            self._bc_socket = _create_receiving_broadcast_socket(self._broadcast_port)
            try:

                ad_timeout = self._next_advertisement_timeout()

                while not current_thread().stopped(): # lifetime loop
                    try:

                        # receive incoming broadcast packets

                        receive_timeout = Timeout(1.0)
                        while not receive_timeout.expired and select([self._bc_socket], [], [], receive_timeout.remain)[0]:
                            try:
                                try:
                                    packet, (client_addr, client_port) = self._bc_socket.recvfrom(57344)
                                except socket_error as e:
                                    if e.args[0] == 10054: # workaround for this issue: http://support.microsoft.com/kb/263823
                                        continue
                                    else:
                                        raise
                                try:
                                    packet = packet.decode("ascii")
                                except UnicodeDecodeError:
                                    continue
                                self._handle_received_packet(packet, client_addr, client_port)
                            except:
                                pmnc.log.error(exc_string()) # log and ignore

                        # send self-advertisement once in a while

                        if ad_timeout.expired:
                            try:
                                self._purge_known_cages()
                                self._broadcast_advertisement()
                            finally:
                                ad_timeout = self._next_advertisement_timeout()

                    except:
                        pmnc.log.error(exc_string()) # log and ignore

            finally:
                self._bc_socket.close()

        except:
            pmnc.log.error(exc_string()) # log and ignore

    ###################################

    # this method overrides identical method from TcpInterface
    # if the interface is configured to use private thread pool

    def _enqueue_request(self, *args, **kwargs):
        self._thread_pool.enqueue(*args, **kwargs)

    ###################################

    def get_cages(self):
        with self._known_cages_lock:
            return set(self._known_cages.keys())

    ###################################

    @typecheck
    def get_nodes(self, cage: valid_cage_name):
        with self._known_cages_lock:
            nodes = self._known_cages.get(cage, {})
            return { node: cage_info["location"]
                     for node, cage_info in nodes.items() }

    ###################################

    def _discovery_response(self, request_id: str) -> bytes:
        return "{0:s}{1:s}:{2:s}://*:{3:d}/".\
               format(self._response_prefix, request_id,
                      self._tcp_interface.encrypted and "ssl" or "tcp",
                      self._tcp_interface.listener_address[1]).\
               encode("ascii")

    ###################################

    # each cage broadcasts its presence, issuing increasing timeouts
    # between the broadcasts, so that all the other cages know soon,
    # but do not receive excessive flood afterwards

    _ad_periods = (10.0, 20.0, 30.0, 60.0, 180.0, 300.0, 600.0) # in seconds, the last one is repeated

    def _next_advertisement_timeout(self) -> Timeout:
        ad_period = self._ad_periods[self._ad_period_idx]
        if self._ad_period_idx < len(self._ad_periods) - 1:
            self._ad_period_idx += 1
        return Timeout(ad_period)

    ###################################

    def _advertisement_packet(self) -> bytes:

        request_id = b2a_hex(urandom(8)).decode("ascii").upper()
        return "{0:s}{1:s}:ADVERTISE-CAGE:{2:s}:{3:s}:{4:d}:{5:s}://*:{6:d}/".\
               format(self._request_prefix, request_id, __node__,
                      self._cage_name, int(self._ad_periods[self._ad_period_idx]),
                      self._tcp_interface.encrypted and "ssl" or "tcp",
                      self._tcp_interface.listener_address[1]).\
               encode("ascii")

    ###################################

    def _broadcast_advertisement(self):

        bc_socket = _create_sending_broadcast_socket(self._bind_address)
        try:
            pmnc.log.debug("broadcasting cage advertisement: {0:s}.{1:s} is at "
                           "{2:s}://*:{3:d}/".format(__node__, self._cage_name,
                           self._tcp_interface.encrypted and "ssl" or "tcp",
                           self._tcp_interface.listener_address[1]))
            bc_socket.sendto(self._advertisement_packet(),
                             (self._broadcast_address, self._broadcast_port))
        finally:
            bc_socket.close()

    ###################################

    def _handle_received_packet(self, packet: str, client_addr: str, client_port: int):

        if not packet.startswith(self._request_prefix):
            return

        request_id, request, request_params = \
            packet[len(self._request_prefix):].split(":", 2)

        if request == "DISCOVER-CAGE": # received discovery request from another cage

            if request_params == self._cage_name:
                discovery_response = self._discovery_response(request_id)
                pmnc.log.debug("sending response to discovery request "
                               "from {0:s}".format(client_addr))
                self._bc_socket.sendto(discovery_response, (client_addr, client_port))

        elif request == "ADVERTISE-CAGE": # received other cage's presence broadcast

            node, cage, period, advertised_location = request_params.split(":", 3)
            try:
                period = int(period)
            except ValueError:
                period = 0

            if valid_cage_name(cage) and valid_node_name(node) and \
               1 <= period <= 3600 and valid_location(advertised_location):

                # the advertisement should contain the URL to the cage,
                # for example ssl://1.2.3.4:5678/ or ssl://*:5678/ the latter
                # form meaning the address from which it has been sent

                cage_addr, cage_port = advertised_location[6:-1].split(":")
                if cage_addr == "*": cage_addr = client_addr
                location = "{0:s}{1:s}:{2:s}/".format(advertised_location[:6],
                                                      cage_addr, cage_port)

                # record the received advertisement, keeping track of time,
                # so that the record expires and disappears without receiving
                # more advertisements

                with self._known_cages_lock:
                    nodes = self._known_cages.setdefault(cage, {})
                    if nodes.get(node, {}).get("location") != location:
                        pmnc.log.debug("received cage advertisement: {0:s}.{1:s} "
                                       "is at {2:s}".format(node, cage, location))
                    nodes[node] = dict(location = location, expires_at = time() + period * 2)

    ###################################

    def _purge_known_cages(self):

        purge_time = time()
        with self._known_cages_lock:
            for cage, nodes in self._known_cages.items():
                expired_nodes = [ node for node, cage_info in nodes.items()
                                  if cage_info["expires_at"]  < purge_time ]
                for expired_node in expired_nodes:
                    del nodes[expired_node]

    ###################################

    def process_rpc_request(self, module, method, args, kwargs):
        return pmnc.__getattr__(module).__getattr__(method)(*args, **kwargs)

###############################################################################

class Handler: # this class is instantiated from interface_tcp

    protocol = "rpc"

    _max_request_size = 1048576

    idle_timeout = 86400.0 # server side of an RPC connection will be kept until
                           # the client closes it, either as idle or expired

    ###################################

    def __init__(self, process_rpc_request, cage_name):
        self._process_rpc_request = process_rpc_request
        self._cage_name = cage_name
        self._rpc_packet = RpcPacket(self._max_request_size)

    ###################################

    @typecheck
    def consume(self, data: bytes) -> bool:
        rpc_request = self._rpc_packet.write(data)
        if rpc_request is not None:
            self._rpc_request = rpc_request # the packet has been received and parsed
            return True
        else:
            return False

    ###################################

    # this method is executed by one of the interface pool threads and performs
    # the actual processing of a previously read and parsed request

    def process_tcp_request(self):

        rpc_request = self._rpc_request

        # extract call information from the dict contained
        # in the received request

        assert rpc_request["target_cage"] == self._cage_name, \
               "expected call to this cage"

        source_cage = rpc_request["source_cage"]
        module = rpc_request["module"]
        method = rpc_request["method"]
        module_method = "{0:s}.{1:s}".format(module, method)
        args = rpc_request["args"]
        kwargs = rpc_request["kwargs"]

        # extract the original request, note that its remaining time
        # is decreased if the current RPC request has less time left

        request_dict = rpc_request["request"]

        request_dict.setdefault("interface", "rpc") # backwards compatibility section
        request_dict.setdefault("protocol", "rpc")
        request_dict["parameters"].setdefault("auth_tokens", {}).\
                                   setdefault("source_cage", source_cage)

        request = pmnc.request.from_dict(request_dict, timeout = pmnc.request.remain)

        # if the connection arrived over SSL, we verify source
        # cage name against the peer's SSL certificate

        auth_tokens = pmnc.request.parameters["auth_tokens"]

        if auth_tokens["encrypted"] and \
           not by_regex("^{0:s}$".format(auth_tokens["peer_cn"]))(source_cage):
            raise Exception("source cage name does not match its SSL certificate")

        # now we know more about the request, it is still the current
        # RPC request, we haven't impersonated the received request yet

        pmnc.request.describe("RPC call {0:s} from {1:s} at {2:s}".\
                              format(module_method, source_cage, auth_tokens["peer_ip"]))
        try:
            pmnc.log.debug("impersonating request {0:s}".format(request.description))
            original_request = current_thread()._request
            current_thread()._request = request
            try:
                result = self._process_rpc_request(module, method, args, kwargs)
            finally:
                current_thread()._request = original_request
        except:
            error = exc_string()
            pmnc.log.error("incoming RPC call failed: {0:s}".format(error))
            rpc_response = dict(exception = error)
            deliver_message = "RPC error"
        else:
            rpc_response = dict(result = result)
            deliver_message = "RPC response, {0:d} byte(s)"

        # marshal the response and prepare for delivery, note that
        # the delivery is done on behalf and within the local request

        response_bytes = RpcPacket.marshal(rpc_response)
        response_hash = sha1(response_bytes).hexdigest().upper()
        response_packet = "{0:08X}{1:s}".\
                          format(len(response_bytes), response_hash).\
                          encode("ascii") + response_bytes
        self._response = BytesIO(response_packet)

        pmnc.log.debug("returning {0:s}".format(
                       deliver_message.format(len(response_packet))))

    ###################################

    @typecheck
    def produce(self, n: int) -> bytes:
        return self._response.read(n)

    ###################################

    @typecheck
    def retract(self, n: int):
        self._response.seek(-n, SEEK_CUR)

###############################################################################

class Resource(TransactionalResource): # RPC resource

    @typecheck
    def __init__(self, name, *,
                 broadcast_address: (str, int),
                 discovery_timeout: float,
                 multiple_timeout_allowance: float,
                 flock_id: valid_flock_id,
                 exact_locations: dict_of(valid_cage_name, valid_exact_location),
                 pool__resource_name: valid_cage_name):

        TransactionalResource.__init__(self, name)

        self._cage_name = pool__resource_name
        self._exact_location = exact_locations.get(self._cage_name)
        self._connect_timeout = discovery_timeout

        if self._exact_location is None:
            broadcast_address, self._broadcast_port = broadcast_address
            self._bind_address, self._broadcast_address = broadcast_address.split("/")
            self._multiple_timeout_allowance = min(multiple_timeout_allowance, 1.0)
            self._request_prefix = "PYTHOMNIC3K-REQUEST:{0:s}:".format(flock_id)
            self._response_prefix = "PYTHOMNIC3K-RESPONSE:{0:s}:".format(flock_id)

    ###################################

    def connect(self):

        TransactionalResource.connect(self)

        # discover and establish connection

        connect_timeout = Timeout(min(self._connect_timeout, pmnc.request.remain))
        if self._exact_location is None:
            self._tcp_resource = self._discover(connect_timeout)
        else:
            self._tcp_resource = self._create_tcp_resource(connect_timeout, self._exact_location)
        self._tcp_resource.connect()

        # if the connection was established over SSL, we verify
        # target cage name against the peer's SSL certificate

        if self._tcp_resource.encrypted and \
           not by_regex("^{0:s}$".format(self._tcp_resource.peer_cn))(self._cage_name):
            raise Exception("target cage name does not match its SSL certificate")

        # reset collected module/method attributes

        self._attrs = []

    ###################################

    def __getattr__(self, name):
        self._attrs.append(name)
        return self

    ###################################

    def __call__(self, *args, **kwargs):

        try:

            attrs, self._attrs = self._attrs, []
            assert len(attrs) == 2, "expected module.method RPC syntax"
            module, method = attrs

            assert not self.resource_args and not self.resource_kwargs, \
                   "synchronous remote calls have no options"

            # wrap up an RPC call and assemble a byte stream

            request_dict = pmnc.request.to_dict()

            # remove request parameters that must not cross the RPC border

            request_dict["parameters"].pop("retry", None)

            # wrap all the call parameters in a plain dict

            request = dict(source_cage = __cage__,
                           target_cage = self._cage_name,
                           module = module, method = method,
                           args = args, kwargs = kwargs,
                           request = request_dict)

            # marshal the call into a packet of bytes

            request_bytes = RpcPacket.marshal(request)
            request_hash = sha1(request_bytes).hexdigest().upper()
            request_packet = "{0:08X}{1:s}".\
                             format(len(request_bytes), request_hash).\
                             encode("ascii") + request_bytes

            self._response = RpcPacket()

            request_description = \
                "RPC request {0:s}.{1:s} to {2:s}, {3:d} byte(s)".\
                format(module, method, self._tcp_resource.server_info,
                       len(request_packet))

            service_request = module == "remote_call" and method == "accept"

        except:
            ResourceError.rethrow(recoverable = True, terminal = False)

        if not service_request: pmnc.log.info("sending {0:s}".format(request_description))
        try:

            response = self._tcp_resource.send_request(request_packet, self.response_handler)
            try:
                result = response["result"]
            except KeyError:
                raise RPCError(description = response["exception"], terminal = False)

        except RPCError as e:
            pmnc.log.warning("{0:s} returned error: {1:s}".\
                             format(request_description, e.description))
            raise
        except:
            pmnc.log.warning("{0:s} failed: {1:s}".\
                             format(request_description, exc_string()))
            raise
        else:
            if not service_request: pmnc.log.info("RPC request returned successfully")
            return result

    ###################################

    def response_handler(self, data: bytes) -> optional(dict):
        return self._response.write(data)

    ###################################

    def disconnect(self):
        try:
            self._tcp_resource.disconnect()
        finally:
            TransactionalResource.disconnect(self)

    ###################################

    def _create_discovery_request(self) -> (bytes, bytes):

        request_id = b2a_hex(urandom(8)).decode("ascii").upper()
        discovery_request = "{0:s}{1:s}:DISCOVER-CAGE:{2:s}".\
                            format(self._request_prefix, request_id, self._cage_name)
        discovery_response_prefix = "{0:s}{1:s}:".format(self._response_prefix, request_id)

        return discovery_request.encode("ascii"), \
               discovery_response_prefix.encode("ascii")

    ###################################

    @typecheck
    def _create_tcp_resource(self, timeout: Timeout, discovered_location: valid_location,
                             remote_addr: optional(str) = None):

        if discovered_location[:6] in ("ssl://", "tcp://") and discovered_location[-1] == "/":
            cage_addr, cage_port = discovered_location[6:-1].split(":")
            if cage_addr == "*":
                assert remote_addr is not None
                cage_addr = remote_addr
        else:
            raise Exception("unsupported RPC protocol")

        if discovered_location.startswith("ssl://"):
            ssl_key_cert_file = _locate_key_file("key_cert.pem")
            ssl_ca_cert_file = _locate_key_file("ca_cert.pem")
        elif discovered_location.startswith("tcp://"):
            ssl_key_cert_file = None
            ssl_ca_cert_file = None

        return pmnc.protocol_tcp.TcpResource(self._cage_name,
                                             server_address = (cage_addr, int(cage_port)),
                                             connect_timeout = timeout.remain,
                                             ssl_key_cert_file = ssl_key_cert_file,
                                             ssl_ca_cert_file = ssl_ca_cert_file)

    ###################################

    def _estimate_node_count(self) -> int:
        rpc_interface = pmnc.interfaces.get_interface("rpc")
        return rpc_interface and len(rpc_interface.get_nodes(self._cage_name)) or 0

    ###################################

    def _discover(self, timeout: Timeout):

        # during discovery we expect to receive as many responses as
        # there are cage instances currently known through advertising

        expected_responses = self._estimate_node_count()

        pmnc.log.debug("discovering cage {0:s}, advertised from {1:d} node(s)".\
                       format(self._cage_name, expected_responses))
        try:

            # if more than one cage is expected to respond, we allow slightly longer
            # initial waiting for responses, controlled by multiple_timeout_allowance

            if expected_responses > 1:
                multiple_timeout_allowance = Timeout(self._multiple_timeout_allowance)

            discovery_request, discovery_response_prefix = self._create_discovery_request()
            received_responses = {}

            # the one-time broadcasting socket must be bound to a random port

            bc_socket = _create_sending_broadcast_socket(self._bind_address)
            try:

                resend_timeout = Timeout(1.0) # broadcast once a second

                while not timeout.expired and not received_responses: # until target cage is discovered or timeout expires

                    # broadcast discovery request

                    bc_socket.sendto(discovery_request, (self._broadcast_address, self._broadcast_port))
                    resend_timeout.reset()

                    # wait for a matching response (there could be more than one)

                    while select([bc_socket], [], [], min(timeout.remain, resend_timeout.remain))[0]:
                        try:

                            packet, (remote_addr, remote_port) = bc_socket.recvfrom(57344)

                            # note that the received packet does not necessarily
                            # contain the discovery response we wait for

                            if remote_port == self._broadcast_port and \
                               packet.startswith(discovery_response_prefix): # but this one is ours

                                # parse the response and append channel instance to the list of discovered

                                discovered_location = packet[len(discovery_response_prefix):].decode("ascii")
                                pmnc.log.debug("received discovery response from {0:s}, cage {1:s} is at {2:s}".\
                                               format(remote_addr, self._cage_name, discovered_location))

                                received_responses[remote_addr] = discovered_location
                                if len(received_responses) >= expected_responses:
                                    break # while select

                            # if some cage instance(s) have already responded, but there are still
                            # more responses to expect, keep waiting within allowance time

                            if received_responses and \
                               (multiple_timeout_allowance.expired or
                                not select([bc_socket], [], [],
                                           min(timeout.remain, resend_timeout.remain,
                                               multiple_timeout_allowance.remain))[0]):
                                break # while select

                        except:
                            pmnc.log.error(exc_string()) # log and ignore

            finally:
                bc_socket.close()

            # if more than one cage instance has been discovered,
            # pick one at random to improve load balancing

            if received_responses:
                remote_addrs = list(received_responses.keys())
                remote_addr = remote_addrs[randint(0, len(remote_addrs) - 1)]
                discovered_location = received_responses[remote_addr]
                tcp_resource = self._create_tcp_resource(timeout, discovered_location, remote_addr)
            else:
                raise Exception("no discovery response from cage {0:s} in {1:.01f} second(s)".\
                                format(self._cage_name, timeout.timeout))

        except:
            pmnc.log.warning("discovery attempt failed: {0:s}".format(exc_string()))
            raise
        else:
            pmnc.log.debug("discovered {0:s}".format(tcp_resource.server_info))
            return tcp_resource

###############################################################################

def self_test():

    from pmnc.request import fake_request
    from time import sleep
    from pmnc.self_test import active_interface
    from pmnc.resource_pool import TransactionExecutionError

    ###################################

    test_interface_config = dict \
    (
    protocol = "rpc",
    random_port = -63000,
    max_connections = 100,
    broadcast_address = ("0.0.0.0/255.255.255.255", 12481),
    flock_id = "SELF_TEST",
    )

    def interface_config(**kwargs):
        result = test_interface_config.copy()
        result.update(kwargs)
        return result

    ###################################

    local_addresses, broadcast_port = test_interface_config["broadcast_address"]
    bind_address, broadcast_address = local_addresses.split("/")

    ###################################

    def test_create_broadcast_socket():

        s1 = _create_receiving_broadcast_socket(broadcast_port)
        s2 = _create_receiving_broadcast_socket(broadcast_port)
        s = _create_sending_broadcast_socket(bind_address)

        p = urandom(64)
        s.sendto(p, (broadcast_address, broadcast_port))

        assert select([s1], [], [], 3.0)[0] == [s1]
        packet = s1.recvfrom(57344)[0]
        assert packet == p

        assert select([s2], [], [], 3.0)[0] == [s2]
        packet = s2.recvfrom(57344)[0]
        assert packet == p

    test_create_broadcast_socket()

    ###################################

    def test_start_stop_interface():

        def process_rpc_request(module, method, args, kwargs):
            pass

        with active_interface("rpc", cage_name = "cage_start_stop", **interface_config(process_rpc_request = process_rpc_request)) as ifc:
            assert ifc.listener_address[0] == bind_address
            assert 63000 <= ifc.listener_address[1] < 64000
            assert ifc._broadcast_port == 12481

    test_start_stop_interface()

    ###################################

    def test_create_discovery_request():

        params = pmnc.config_resource_rpc.copy()
        del params["protocol"]           # normally resource.py does this
        del params["pool__idle_timeout"] # normally transaction.py does this
        del params["pool__max_age"]
        r = Resource("test", pool__resource_name = "cage123", **params)

        rq, rsp = r._create_discovery_request()
        rq = rq.decode("ascii"); rsp = rsp.decode("ascii")
        assert by_regex("^PYTHOMNIC3K-REQUEST:SELF_TEST:[0-9A-F]{16}:DISCOVER-CAGE:cage123$")(rq)
        rid = rq.split(":", 4)[2]
        assert rsp == "PYTHOMNIC3K-RESPONSE:SELF_TEST:{0:s}:".format(rid)

    test_create_discovery_request()

    ###################################

    def test_discover_once():

        def process_rpc_request(module, method, args, kwargs):
            pass

        with active_interface("rpc", **interface_config(process_rpc_request = process_rpc_request,
                              cage_name = "cage_discover_once")) as ifc:

            s = _create_sending_broadcast_socket(bind_address)

            request_id = b2a_hex(urandom(8)).decode("ascii").upper()
            request = "PYTHOMNIC3K-REQUEST:SELF_TEST:{0:s}:DISCOVER-CAGE:{1:s}".\
                      format(request_id, "cage_discover_once").encode("ascii")
            response = "PYTHOMNIC3K-RESPONSE:SELF_TEST:{0:s}:ssl://*:{1:d}/".\
                       format(request_id, ifc.listener_address[1]).encode("ascii")

            for i in range(3):
                s.sendto(request, (broadcast_address, ifc._broadcast_port))
                sleep(0.5)

            t = Timeout(3.0)
            while not t.expired and select([s], [], [], t.remain)[0]:
                packet = s.recvfrom(57344)[0]
                if packet == response:
                    break
            else:
                raise Exception("no discovery response")

    test_discover_once()

    ###################################

    def test_advertisement():

        def process_rpc_request(module, method, args, kwargs):
            pass

        with active_interface("rpc", **interface_config(process_rpc_request = process_rpc_request,
                              cage_name = "cage_advertisement", ad_periods = (1.0, ))) as ifc:

            assert not ifc.get_cages()
            assert ifc.get_nodes("cage_advertisement") == {}

            timeout = Timeout(5.0)
            while not timeout.expired:
                nodes = ifc.get_nodes("cage_advertisement")
                if nodes:
                    assert len(nodes) == 1
                    loc = nodes["self_test"]
                    assert valid_location(loc)
                    assert loc.endswith(":{0:d}/".format(ifc.listener_address[1]))
                    break
                sleep(1.0)
            else:
                assert False, "should have advertised itself"

        sleep(3.0)
        ifc._purge_known_cages()
        assert ifc.get_cages() == { "cage_advertisement" }
        assert ifc.get_nodes("cage_advertisement") == {}

    test_advertisement()

    ###################################

    def test_exact_location():

        def process_rpc_request(module, method, args, kwargs):
            return "been there"

        with active_interface("rpc", **interface_config(process_rpc_request = process_rpc_request,
                              cage_name = "cage_exact_location", ad_periods = (1.0, ))) as ifc:

            fake_request(10.0)

            # wait for the cage to be advertised

            nodes = ifc.get_nodes("cage_exact_location")
            while not nodes and not pmnc.request.expired:
                sleep(1.0)
                nodes = ifc.get_nodes("cage_exact_location")
            assert nodes, "should have advertised itself"

            # then initiate connection to the now known exact address

            exact_locations = { "cage_exact_location": nodes["self_test"] }

            resource = pmnc.protocol_rpc.Resource("self_test.cage_exact_location",
                                                  broadcast_address = ("UNUSED/UNUSED", 0),
                                                  discovery_timeout = 3.0,
                                                  multiple_timeout_allowance = 0.0,
                                                  flock_id = "UNUSED",
                                                  exact_locations = exact_locations, # same as in config file
                                                  pool__resource_name = "cage_exact_location")
            resource.connect()
            try:
                resource.begin_transaction("xid", source_module_name = __name__,
                                           transaction_options = {}, resource_args = (),
                                           resource_kwargs = {})
                try:
                    result = resource.module.method()
                except:
                    resource.rollback()
                    raise
                else:
                    resource.commit()
            finally:
                resource.disconnect()

            assert result == "been there"

    test_exact_location()

    ###################################

    def test_resource_success():

        def process_rpc_request(module, method, args, kwargs):
            return eval(args[0])

        with active_interface("rpc", **interface_config(process_rpc_request = process_rpc_request,
                              cage_name = "cage_resource_success")) as ifc:

            fake_request(10.0)
            pmnc.request.describe("it's my request all right")

            xa = pmnc.transaction.create()
            xa.rpc__cage_resource_success.foo.bar("module, method, args, kwargs", "param", a = "b")
            xa.rpc__cage_resource_success.biz.baz("pmnc.request.to_dict()")
            r1, r2 = xa.execute()

            assert r1 == ("foo", "bar", ("module, method, args, kwargs", "param"), { "a": "b" })

            deadline = r2.pop("deadline")
            assert abs(deadline - (time() + pmnc.request.remain)) < 0.01

            assert r2 == dict(protocol = pmnc.request.protocol,
                              description = "it's my request all right",
                              parameters = dict(auth_tokens = { "source_cage": __cage__ }),
                              interface = pmnc.request.interface,
                              unique_id = pmnc.request.unique_id)

    test_resource_success()

    ###################################

    def test_resource_failure():

        def process_rpc_request(module, method, args, kwargs):
            1 / 0

        with active_interface("rpc", **interface_config(process_rpc_request = process_rpc_request,
                              cage_name = "cage_resource_failure")) as ifc:

            fake_request(10.0)

            xa = pmnc.transaction.create()
            xa.rpc__cage_resource_failure.foo.bar()
            try:
                xa.execute()
            except RPCError as e:
                assert e.description.startswith("ZeroDivisionError")
            else:
                assert False

    test_resource_failure()

    ###################################

    def test_resource_local_timeout():

        def process_rpc_request(module, method, args, kwargs):
            assert 1.5 < pmnc.request.remain < 2.5 # interface timeout takes over the received request timeout
            sleep(3.0)

        with active_interface("rpc", **interface_config(process_rpc_request = process_rpc_request,
                              cage_name = "cage_resource_local_timeout", request_timeout = 2.0)) as ifc:

            fake_request(5.0)

            xa = pmnc.transaction.create()
            xa.rpc__cage_resource_local_timeout.foo.bar()
            try:
                xa.execute()
            except ResourceError as e:
                assert str(e).startswith("unexpected eof reading data from cage_resource_local_timeout") # as interface drops the connection
            else:
                assert False

    test_resource_local_timeout()

    ###################################

    def test_resource_remote_timeout():

        def process_rpc_request(module, method, args, kwargs):
            assert 2.5 < pmnc.request.remain < 3.5 # inherited remote timeout
            sleep(4.0)

        with active_interface("rpc", **interface_config(process_rpc_request = process_rpc_request,
                              cage_name = "cage_resource_remote_timeout")) as ifc:

            fake_request(5.0)
            sleep(2.0)

            xa = pmnc.transaction.create()
            xa.rpc__cage_resource_remote_timeout.foo.bar()
            try:
                xa.execute()
            except (ResourceError, TransactionExecutionError) as e:
                assert str(e).startswith("request deadline reading data from cage_resource_remote_timeout") or \
                       str(e).startswith("request deadline waiting for intermediate result from resource rpc__cage_resource_remote_timeout")
            else:
                assert False

    test_resource_remote_timeout()

    ###################################

    def test_pmnc_discovery():

        fake_request(5.0)

        try:
            pmnc("never_existed").foo.bar()
        except ResourceError as e:
            assert str(e) == "no discovery response from cage never_existed in 3.0 second(s)"
            assert e.recoverable and e.terminal
        else:
            assert False

    test_pmnc_discovery()

    ###################################

    def test_pmnc_success():

        def process_rpc_request(module, method, args, kwargs):
            return module, method, args, kwargs, pmnc.request.to_dict()

        with active_interface("rpc", **interface_config(process_rpc_request = process_rpc_request,
                              cage_name = "cage_pmnc_success")) as ifc:
            fake_request(5.0)
            pmnc.request.describe("yes it is")
            result = pmnc("cage_pmnc_success").foo.bar(1, "foo", biz = "baz")

        module, method, args, kwargs, request = result
        assert module == "foo" and method == "bar" and args == (1, "foo") and kwargs == { "biz": "baz" }

        deadline = request.pop("deadline")
        assert abs(deadline - (time() + pmnc.request.remain)) < 0.01

        assert request == dict \
                          (
                              protocol = pmnc.request.protocol,
                              description = "yes it is",
                              parameters = dict(auth_tokens = { "source_cage": __cage__ }),
                              interface = pmnc.request.interface,
                              unique_id = pmnc.request.unique_id,
                          )

    test_pmnc_success()

    ###################################

    def test_pmnc_failure():

        def process_rpc_request(module, method, args, kwargs):
            {}["not-there"]

        with active_interface("rpc", **interface_config(process_rpc_request = process_rpc_request,
                              cage_name = "cage_pmnc_failure")) as ifc:
            fake_request(5.0)
            try:
                pmnc("cage_pmnc_failure").foo.bar()
            except RPCError as e:
                assert e.description.startswith("KeyError(\"'not-there'\") in process_rpc_request()")
                assert not e.recoverable and not e.terminal
            else:
                assert False

    test_pmnc_failure()

    ###################################

    def test_two_calls():

        def process_rpc_request1(module, method, args, kwargs):
            assert pmnc.request.parameters["auth_tokens"]["source_cage"] == __cage__
            pmnc.request.parameters["auth_tokens"]["source_cage"] = "set once"
            return pmnc("cage_second").module.second()

        def process_rpc_request2(module, method, args, kwargs):
            return pmnc.request.to_dict()

        with active_interface("rpc1", **interface_config(process_rpc_request = process_rpc_request1,
                              cage_name = "cage_first")) as ifc1:
            with active_interface("rpc2", **interface_config(process_rpc_request = process_rpc_request2,
                                  cage_name = "cage_second")) as ifc2:
                fake_request(10.0)
                result = pmnc("cage_first").foo.bar()

        deadline = result.pop("deadline")
        assert abs(deadline - (time() + pmnc.request.remain)) < 0.01

        assert result == dict(protocol = pmnc.request.protocol,
                              description = None,
                              parameters = dict(auth_tokens = { "source_cage": "set once" }),
                              interface = pmnc.request.interface,
                              unique_id = pmnc.request.unique_id)

    test_two_calls()

    ###################################

if __name__ == "__main__": import pmnc.self_test; pmnc.self_test.run()

###############################################################################
# EOF


import json
import os
import time
start = time.time()

def get_input(fname):
    fo = open(r".\count\\" + fname.split("\\")[-1], "w", encoding="UTF-8")
    with open(fname, "r", encoding="utf-8-SIG") as load_f:
        load_dict = json.load(load_f)
        for i in load_dict:
            if "NotSupported" in i and "dotnet" in i["NotSupported"]:
                fo.write(i["Input"]+"\n")
    fo.close

def get_all_dir(path):
    filess = os.listdir(path)
    print(filess)
    for files in filess:
        if files != "README.md":
            if os.path.isdir(path+files):
                print("dir:"+files)
                get_all_dir(path+files+'\\')
            else:
                get_input(path+"\\"+files)

path = r".\Specs\DateTime\Spanish\\"
getAllDir(path)
end = time.time()
print(end-start)


# -*- coding: utf-8 -*-
"""
Created on Mon May 13 10:42:54 2019

@author: Administrator
"""

import requests

api_key = "_your_api_key_"

url = "https://goo.gl/maps/QGs5TLGaAjdaG6oN8"

center = "Dehradun"
zoom = 10

r = requests.get(url + "center =" + center + "&zoom =" +
                   str(zoom) + "&size = 400x400&key =" +
                             api_key + "sensor = false")

f = open('address of the file location ', 'wb')
f.write(r.content)
f.close()