import numpy as np
import math
import cv2
import matplotlib.pyplot as plt
import matplotlib.patches as patches
import logging

from .score_semantic import ScoreSemantic
from .chromaticChannels import composeChromaticMap

def scoreAnno (heatmap, srcimg, labels, target=[]):
	#print(f'heatmap:{heatmap.shape} srcimg{srcimg.shape}')

	labels_count = len(labels)
	#labels_count = min(labels_count, 4)

	jout = ScoreSemantic(heatmap, labels)
	#print(jout.dump_json())
	marks = jout.marks
	# print(f"jout.marks: {len(marks)}")
	clean_bgr = np.zeros_like(heatmap[0])

	if len(target) > 0:
		_, ax = plt.subplots(labels_count+2, 3)
	else:
		_, ax = plt.subplots(labels_count+2, 2)

	for i in range(labels_count):
		ax[i][1].imshow(clean_bgr)

		if marks[i].get('points') is not None:
			points = marks[i]['points']
			if len(points) > 0:
				ax[i][1].plot(points[:, 0], points[:, 1], 'g.')
		elif marks[i].get('vlines') is not None:
			lines = marks[i]['vlines']
			if len(lines) > 0:
				ax[i][1].vlines(lines[:, 0], lines[:, 1], lines[:, 2], color = 'g')
		elif marks[i].get('rectangles') is not None:
			rectangles = marks[i]['rectangles']
			if len(rectangles) > 0:
				for rect in rectangles:
					rc = patches.Rectangle((rect[0], rect[1]), rect[2], rect[3], linewidth=1, edgecolor='b', facecolor='g')
					ax[i][1].add_patch(rc)
		elif marks[i].get('boxes') is not None:
			boxes = marks[i]['boxes']
			if len(boxes) > 0:
				for box in boxes:
					#print('box:', box)
					angle = box[2] * math.pi / 180
					wcos = box[1][0] * math.cos(angle)
					wsin = box[1][0] * math.sin(angle)
					hcos = box[1][1] * math.cos(angle)
					hsin = box[1][1] * math.sin(angle)
					rx = wcos - hsin
					ry = wsin + hcos
					rc = patches.Rectangle((box[0][0] - rx / 2, box[0][1] - ry / 2), box[1][0], box[1][1], angle=box[2], linewidth=1, edgecolor='b', facecolor='g')
					ax[i][1].add_patch(rc)

		ax[i][1].set_ylabel(labels[i], fontsize='small', rotation='horizontal')
		ax[i][0].imshow(heatmap[i],cmap='jet')

		hm_max = np.max(heatmap[i])
		ax[i][0].set_ylabel(f'({hm_max})', fontsize='small', rotation='horizontal')

		if len(target) > 0:
			ax[i][2].imshow(target[...,i], cmap='jet')

	ax[labels_count][1].imshow(clean_bgr)

	all_points = [m.get('points') for m in marks if len(m.get('points', [])) > 0]
	if len(all_points) > 0:
		all_points = np.concatenate(all_points)
		print(f'all_points-{all_points.shape} heatmap-{heatmap.shape}')
		ax[labels_count][1].plot(all_points[:, 0], all_points[:,1], 'g.')

	ax[labels_count][1].set_ylabel('max', fontsize='small', rotation='horizontal')
	ax[labels_count][0].imshow(np.max(heatmap, axis=0), cmap='jet')
	if len(target) > 0:
		ax[labels_count][2].imshow(np.max(target, axis=-1), cmap='jet')
		ax[labels_count+1][2].imshow(srcimg, cmap='gray')

	ax[labels_count+1][1].set_ylabel('srcimg', fontsize='small', rotation='horizontal')
	ax[labels_count+1][1].imshow(srcimg, cmap='gray')
	#ax[labels_count+1][0].imshow(srcimg, cmap='gray')
	ax[labels_count+1][0].imshow(srcimg)
	plt.get_current_fig_manager().full_screen_toggle()
	plt.show()
	#plt.close()

def scoreAnnoChromatic (feature, targets, predictions=None):
	#print('scoreAnnoChromatic:', feature.shape, targets.shape, predictions.shape)
	with_pred = predictions is not None
	_, ax = plt.subplots(2, 2)

	colored_target = composeChromaticMap(targets)
	cv2.imwrite('./temp/feature.png', feature)
	cv2.imwrite('./temp/target.png', colored_target)

	#print('colored_target:', colored_target.shape, targets.dtype)
	ax[0][0].imshow(colored_target, cmap='gray')
	if with_pred:
		colored_pred = composeChromaticMap(predictions)
		ax[0][1].imshow(colored_pred)
		cv2.imwrite('./temp/pred.png', colored_pred)

	ax[1][0].imshow(feature)
	ax[1][1].set_ylabel('feature', rotation='vertical')
	ax[1][1].imshow(feature, cmap='gray')

	plt.get_current_fig_manager().full_screen_toggle()
	plt.show()
	#plt.close()

def gaugeToRGB (gauge, frac_y = False):	# gauge: [Y(h, w), K(h, w)]
	mapy = gauge[0] * 8 + 128
	mapk = gauge[1] * 127 + 128

	result = None
	if frac_y:
		B, R = np.modf(mapy)
		result = np.stack([B * 256, mapk, R], axis = 2)
	else:
		result = np.stack([np.zeros(mapy.shape, np.float32), mapk, mapy], axis = 2)
	result = np.uint8(np.clip(result, 0, 255))

	return result

image_staff_frame = None

def gaugeToFrame (gauge):	# gauge: [Y(h, w), K(h, w)]
	global image_staff_frame
	if image_staff_frame is None:
		image_staff_frame = cv2.imread('./assets/staff-frame.png')
		image_staff_frame = image_staff_frame[:, :, ::-1]

	# integrate to make X map
	map_x = np.zeros(gauge.shape[1:], np.float32)
	half_height = gauge.shape[1] // 2
	for x in range(gauge.shape[2]):
		map_x[half_height, x] = x + 100

	for y in range(half_height):	# X[y - 1] = X[y] - K[y]
		map_x[half_height - y - 1, :] = map_x[half_height - y, :] - gauge[1, half_height - y, :]

	for y in range(1, half_height):	# X[y] = X[y - 1] + K[y]
		map_x[half_height + y, :] = map_x[half_height + y - 1, :] + gauge[1, half_height + y, :]

	map_y = gauge[0] * 8 + image_staff_frame.shape[0] // 2

	return cv2.remap(image_staff_frame, map_x, map_y, cv2.INTER_CUBIC, borderMode = cv2.BORDER_REPLICATE)

def gaugeToFrameFeature (gauge, feature):
	frame = gaugeToFrame(gauge).astype(np.float32)

	inv_feature = 255 - np.minimum(np.float32(feature) + 64, 255)
	inv_feature = np.clip(inv_feature, 0, 255)

	inv_feature = np.tile(inv_feature[:, :, None], (1, 1, 3))

	return (frame + inv_feature).clip(0, 255).astype(np.uint8)

def scoreAnnoGauge (feature, targets, predictions=None):
	with_pred = predictions is not None
	fig, ax = plt.subplots(4, 2)

	cv2.imwrite('./temp/feature.png', feature)
	cv2.imwrite('./temp/target.png', gaugeToRGB(targets))

	if len(targets) >= 3:
		ax[0][0].imshow(targets[2])
	else:
		ax[0][0].imshow(feature)
	ax[0][1].set_ylabel('feature', rotation='vertical')
	ax[0][1].imshow(feature, cmap='gray')

	ax[1][1].set_ylabel('Y')
	ax[2][1].set_ylabel('K')

	y = ax[1][0].imshow(targets[0])
	k = ax[2][0].imshow(targets[1])
	fig.colorbar(y, ax = ax[1][0], shrink=0.4, pad=0.01)
	fig.colorbar(k, ax = ax[2][0], shrink=0.4, pad=0.01)

	frame = gaugeToFrameFeature(targets, feature)
	ax[3][0].imshow(frame)

	if with_pred:
		y = ax[1][1].imshow(predictions[0])
		k = ax[2][1].imshow(predictions[1])
		fig.colorbar(y, ax = ax[1][1], shrink=0.4, pad=0.01)
		fig.colorbar(k, ax = ax[2][1], shrink=0.4, pad=0.01)
		cv2.imwrite('./temp/pred.png', gaugeToRGB(predictions))

		pred_frame = gaugeToFrameFeature(predictions, feature)
		ax[3][1].imshow(pred_frame)

	plt.get_current_fig_manager().full_screen_toggle()
	plt.show()
	#plt.close()

class DatasetViewer:
	def __init__(self, config, chromatic=True, gauge_mode=False):
		self.chromatic = chromatic
		self.gauge_mode = gauge_mode

		self.labels = config['data.args.labels']

	def show(self, data_set):
		for batch, (feature, target) in enumerate(data_set):
			bsize = len(feature)
			for i in range(bsize):
				index = batch * bsize + i
				logging.info(index)

				img = feature[i].cpu().numpy()
				hm = target[i].cpu().numpy()
				img = np.moveaxis(img, 0, -1)
				img = img.reshape(img.shape[:2]) if img.shape[2] == 1 else img.reshape(img.shape[:3])
				img = np.clip(img, 0., 1.)
				img = np.uint8(img * 255)
				#print('img shape2:', img.shape)

				if self.gauge_mode:
					scoreAnnoGauge(img, hm)
				else:
					if self.chromatic:
						scoreAnnoChromatic(img, hm)
					else:
						hm = np.uint8(hm * 255)
						scoreAnno(hm, img, labels = self.labels)


from django.urls import path

from chatapi.views import views

urlpatterns = [
    path('messages/<str:user_id>/<int:thread_id>/', views.messages),
    path('message/<str:user_id>/<int:thread_id>/<int:sequential_order>', views.message),
    path('chat/', views.chat)
]

# views = NinjaAPI(csrf=True)
# views.add_router('auth/', auth_api)
# views.add_router('', chat_api)
#
# urlpatterns = [
#     path("views/", views.urls, name="views"),
#     path("", redirect_to_docs, name="index"),
# ]


from werkzeug.utils import secure_filename
from .sequencer.task_impl import ADSTask, ADSSequence
from .sequencer.task_state import TaskState
from .event_manager import EventManager
from .factory import db
from .models import Snapshot
from .snapshot_log import checkout

event_manager = EventManager()
event_manager.start()

sequence_to_execute = None

def checkout_sections(selected_sections):
    newest_snapshot = Snapshot.query.order_by(
        Snapshot.timestamp.desc()
    ).limit(1).first()

    data = newest_snapshot.to_json()
    diffs = checkout(data, selected_sections)
    return diffs

def remove_dict_null(data):
    copied_keys = tuple(data.keys())
    for key in copied_keys:
        if isinstance(data[key], dict) and data[key]:
            remove_dict_null(data[key])
        if not data[key]:
            data.pop(key)

def save_timing(timing_data):
    for dutyfactor in timing_data:
        for acct in ['ACCT1', 'ACCT2', 'ACCT3', 'ACCT4']:
            if timing_data[dutyfactor]['count'] == 0:
                timing_data[dutyfactor][acct] = 0
            else:
                timing_data[dutyfactor][acct] /= timing_data[dutyfactor]['count']

        timing_item = Timing(
            dutyfactor=dutyfactor,
            times=timing_data[dutyfactor]['beam_time'],
            acct1=timing_data[dutyfactor]['ACCT1'],
            acct2=timing_data[dutyfactor]['ACCT2'],
            acct3=timing_data[dutyfactor]['ACCT3'],
            acct4=timing_data[dutyfactor]['ACCT4'],
        )
        db.session.add(timing_item)
    db.session.commit()

def create_sequence(data, sequence_name, task_level):
    children = []
    tasks = data[sequence_name]["tasks"]
    next_task_level = task_level + 1
    for task in tasks:
        task_id = task["task_id"]
        child = data[task_id]
        if child["task_type"] == "task":
            children.append(Task(
                task_type="task",
                name=child["name"],
                description=child["description"],
                skippable=child["skippable"],
                interactive=child["interactive"],
                user_code=child["user_code"],
            ))
        else:
            sequence = Task(
                task_type="seq",
                name=child["name"],
                description=child["description"],
                task_level=task_level,
                children=create_sequence(data, task_id, next_task_level)
            )
            children.append(sequence)
    return children

def insert_tasks():
    with open('sequencer/task_data.json') as f:
        data = json.load(f)
    for task_name in data:
        if data[task_name]["task_type"] == "seq":
            sequence = Task(
                task_type="seq",
                name=data[task_name]["name"],
                description=data[task_name]["description"],
                task_level=0,
                children=create_sequence(data, task_name, task_level=1)
            )
            db.session.add(sequence)
    db.session.commit()

def execute_sequence(sequence_name):
    server = None
    tasks = []
    seq = db.session.query(Task).filter(Task.name==sequence_name).first()
    tasks.append(sequence)
    subtasks = subtasks_instantiate(seq, server)
    tasks.extend(subtasks)
    for t in tasks:
        if isinstance(t, Sequence):
            continue
        t.userCode.execUserCode()


import json
from dateutil.parser import parse as date_parser

from django.apps import apps
from django.shortcuts import HttpResponse
from django.template.response import TemplateResponse
from django.views.decorators.gzip import gzip_page
from django.views.decorators.clickjacking import xframe_options_exempt
from rest_framework.exceptions import ValidationError

from surf.apps.core.search import get_search_client
from surf.apps.materials.serializers import SearchSerializer
from surf.apps.materials.utils import add_extra_parameters_to_materials

filters_app = apps.get_app_config("filters")

@xframe_options_exempt
@gzip_page
def widget_iframe_content(request):
    # Validates incoming data
    data = request.GET.dict()
    if "search_text" in data:
        data["search_text"] = data["search_text"] if data["search_text"] != '""' else ""
    if "filters" in data:
        data["filters"] = json.loads(data["filters"])
    serializer = SearchSerializer(data=data)
    try:
        serializer.is_valid(raise_exception=True)
    except ValidationError as exc:
        return HttpResponse(exc, status=400)
    data = serializer.validated_data
    # Transforms incoming data for search request
    data["filters"] = [
        {"external_id": external_id, "items": items}
        for external_id, items in data["filters"].items()
    ]
    data["drilldown_names"] = filters_app.metadata.get_filter_field_names()
    client = get_search_client()
    res = client.search(**data)
    records = res["records"]
    for record in records:
        if not record["published_at"]:
            continue
        record["published_at"] = date_parser(record["published_at"])
    records = add_extra_parameters_to_materials(filters_app.metadata, records)
    return TemplateResponse(
        request=request,
        template="widget/index.html",
        context={
            "records": records,
            "record_count": res["recordcount"],
            "technical_type_translations": {
                technical_type: translations[request.LANGUAGE_CODE]
                for technical_type, translations in filters_app.metadata.translations["technical_type"].items()
            },
        }
    )


import os
import fitz

def get_file(pdfpath):
    files = os.listdir(pdfpath)  # 默认访问的是当前路径
    lis = [pdfpath + os.sep + file for file in files if os.path.splitext(file)[1] == '.pdf']
    return lis

def conver_one_img(pdf_path):
    with fitz.open(pdf_path) as doc:
            pdf_name = os.path.basename(pdf_path).split('.')[0]

            for pg in range(doc.pageCount):
                page = doc[pg]
                rotate = int(0)
                # 每个尺寸的缩放系数为10，这将为我们生成分辨率提高100倍的图像。
                zoom_x, zoom_y = 1.5, 1.5
                trans = fitz.Matrix(zoom_x, zoom_y).preRotate(rotate)
                pm = page.getPixmap(matrix=trans, alpha=False)
                # pm.writePNG('%s.png' % pdf_name)
                pm.writePNG(f'./img/{pdf_name}.png')

def conver_img(pdf_dir):
    for pdf in pdf_dir:
        # doc = fitz.open(pdf)
        with fitz.open(pdf) as doc:
            pdf_name = os.path.basename(pdf).split('.')[0]

            for pg in range(doc.pageCount):
                page = doc[pg]
                rotate = int(0)
                # 每个尺寸的缩放系数为10，这将为我们生成分辨率提高100倍的图像。
                zoom_x, zoom_y = 1.5, 1.5
                trans = fitz.Matrix(zoom_x, zoom_y).preRotate(rotate)
                pm = page.getPixmap(matrix=trans, alpha=False)
                # pm.writePNG('%s.png' % pdf_name)
                pm.writePNG(f'./img/{pdf_name}.png')

def main():
    pdfpath = r'./page'
    # pdfpath = '.'
    pdf_dir = get_file(pdfpath)
    conver_img(pdf_dir)

if __name__ == '__main__':
    main()


import torch
from torch.nn import ZeroPad2d
from torch import LongTensor
from torch.utils.data import random_split
from torch import count_nonzero
import math
from collections import deque
from treelib import Tree, Node
from sklearn.model_selection import train_test_split
from itertools import zip_longest
import networkx as nx
from treelib import Tree, Node
import numpy as np
from itertools import compress, islice
import os
from pathlib import Path
import json

from data.tokens import grouper_mol

DATA_DIR = "resource"
NODE_TYPE_DICT = {'F': 9, 'O': 10, 'N': 11, 'C': 12, 'P': 13, 'I': 14, 'Cl': 15, 'Br': 16, 'S': 17}
TYPE_NODE_DICT = {str(key): value for value, key in NODE_TYPE_DICT.items()}
BOND_TYPE_DICT = {1: 5, 2: 6, 3: 7, 1.5: 8}
TYPE_BOND_DICT = {key: value for value, key in NODE_TYPE_DICT.items()}

def get_level(node):
    return int(node.identifier.split('-')[0])

def get_location(node):
    return int(node.identifier.split('-')[1])

def get_k(tree):
    return math.sqrt(len(tree[tree.root].successors(tree.identifier)))

def get_parent(node, tree):
    return tree[node.predecessor(tree.identifier)]

def get_children_identifier(node, tree):
    return sorted(node.successors(tree.identifier), key=lambda x: get_sort_key(x))

def get_sort_key(node_id):
    if len(node_id.split('-')) > 2:
        return (int(node_id.split('-')[0]), int(node_id.split('-')[1]), int(node_id.split('-')[2]))
    else:
        return (int(node_id.split('-')[0]), int(node_id.split('-')[1]))

def nearest_power(N, base=2):
    a = int(math.log(N, base))
    if base**a == N:
        return N

    return base**(a + 1)

def adj_to_k2_tree(adj, return_tree=False, is_wholetree=False, k=4, is_mol=False):
    if not is_mol:
        adj[adj > 0] = 1
    n_org_nodes = adj.shape[0]
    # add padding (proper size for k)
    n_nodes = nearest_power(n_org_nodes, k)
    k_square = k**2
    padder = ZeroPad2d((0, n_nodes-n_org_nodes, 0, n_nodes-n_org_nodes))
    padded_adj = padder(adj)
    total_level = int(math.log(n_nodes, k))
    tree_list = []
    leaf_list = []
    tree = Tree()
    # add root node
    tree.create_node("root", "0")
    tree_key_list = deque([])
    slice_size = int(n_nodes / k)
    # slice matrices
    start_index = range(0,n_nodes,slice_size)
    end_index = range(slice_size,n_nodes+1,slice_size)
    slices = []
    for row_start, row_end in zip(start_index, end_index):
        for col_start, col_end in zip(start_index, end_index):
            slices.append(padded_adj[row_start:row_end, col_start:col_end])

    sliced_adjs = deque(slices)
    sliced_adjs_is_zero = LongTensor([int(count_nonzero(adj)>0) for adj in sliced_adjs])
    tree_list.append(sliced_adjs_is_zero)
    # molecule + only leaf
    if is_mol and adj.shape[0] == k:
        tree_element_list = deque(list(map(int, torch.flatten(adj).tolist())))
    else:
        tree_element_list = deque(sliced_adjs_is_zero)

    for i, elem in enumerate(tree_element_list, 1):
        tree.create_node(elem, f"1-{i}", parent="0")
        tree_key_list.append(f"1-{i}")

    while (slice_size != 1):
        n_nodes = sliced_adjs[0].shape[0]
        if n_nodes == k:
            if is_wholetree:
                leaf_list = [adj.reshape(k_square,) for adj in sliced_adjs]
            else:
                leaf_list = [adj.reshape(k_square,) for adj in sliced_adjs if count_nonzero(adj)>0]
            break
        slice_size = int(n_nodes / k)
        target_adj = sliced_adjs.popleft()
        target_adj_size = target_adj.shape[0]
        if return_tree:
            parent_node_key = tree_key_list.popleft()
        # remove adding leaves to 0
        if not is_wholetree:
            if count_nonzero(target_adj) == 0:
                continue
        # generate tree_list and leaf_list
        new_sliced_adjs = []
        start_index = range(0,n_nodes,slice_size)
        end_index = range(slice_size,n_nodes+1,slice_size)
        for row_start, row_end in zip(start_index, end_index):
            for col_start, col_end in zip(start_index, end_index):
                new_sliced_adjs.append(target_adj[row_start:row_end, col_start:col_end])
        new_sliced_adjs_is_zero = LongTensor([int(count_nonzero(adj)>0) for adj in new_sliced_adjs])
        sliced_adjs.extend(new_sliced_adjs)
        tree_list.append(new_sliced_adjs_is_zero)

        if return_tree:
            # generate tree
            tree_element_list.extend(new_sliced_adjs_is_zero)
            cur_level = int(total_level - math.log(target_adj_size, k) + 1)
            cur_level_key_list = [int(key.split('-')[1]) for key in tree_key_list if int(key.split('-')[0]) == cur_level]
            if len(cur_level_key_list) > 0:
                key_starting_point = max(cur_level_key_list)
            else:
                key_starting_point = 0
            for i, elem in enumerate(new_sliced_adjs_is_zero, key_starting_point+1):
                tree.create_node(elem, f"{cur_level}-{i}", parent=parent_node_key)
                tree_key_list.append(f"{cur_level}-{i}")

    if return_tree:
        # add leaves to tree
        leaves = [node for node in tree.leaves() if node.tag == 1]
        index = 1
        for leaf, leaf_values in zip(leaves, leaf_list):
            for value in leaf_values:
                tree.create_node(int(value), f"{total_level}-{index}", parent=leaf)
                index += 1
        return tree
    else:
        return tree_list, leaf_list

def check_tree_validity(tree):
    depth = tree.depth()
    if depth == 1:
        return False
    leaves = [leaf for leaf in tree.leaves() if leaf.tag != '0']
    invalid_leaves = [leaf for leaf in leaves if tree.depth(leaf)!=depth]
    if len(invalid_leaves) == 0:
        return True
    else:
        return False

def tree_to_adj(tree, k=2):
    '''
    convert k2 tree to adjacency matrix
    '''
    tree = map_starting_point(tree, k)
    depth = tree.depth()
    leaves = [leaf for leaf in tree.leaves() if leaf.tag != '0']
    one_data_points = [leaf.data for leaf in leaves]
    x_list = [data[0] for data in one_data_points]
    y_list = [data[1] for data in one_data_points]
    label_list = [NODE_TYPE_DICT[leaf.tag] if leaf.tag in NODE_TYPE_DICT.keys() else int(leaf.tag) for leaf in leaves]
    matrix_size = int(k**depth)
    adj = torch.zeros((matrix_size, matrix_size))
    for x, y, label in zip(x_list, y_list, label_list):
        # if (x > len(adj)) or (y > len(adj)):
        #     return None
        adj[x, y] = label

    return adj

def map_starting_point(tree, k):
    '''
    map starting points for each elements in tree (to convert adjacency matrix)
    '''
    try:
        bfs_list = [tree[node] for node in tree.expand_tree(mode=Tree.WIDTH,
                                                            key=lambda x: (int(x.identifier.split('-')[0]), int(x.identifier.split('-')[1]), int(x.identifier.split('-')[2])))]
    except:
        bfs_list = [tree[node] for node in tree.expand_tree(mode=Tree.WIDTH,
                                                            key=lambda x: (int(x.identifier.split('-')[0]), int(x.identifier.split('-')[1])))]
    bfs_list[0].data = (0,0)

    for node in bfs_list[1:]:
        parent = get_parent(node, tree)
        siblings = get_children_identifier(parent, tree)
        index = siblings.index(node.identifier)
        level = get_level(node)
        tree_depth = tree.depth()
        matrix_size = k**tree_depth
        adding_value = int(matrix_size/(k**level))
        parent_starting_point = parent.data
        node.data = (parent_starting_point[0]+adding_value*int(index/k), parent_starting_point[1]+adding_value*int(index%k))

    return tree

def map_child_deg(node, tree):
    '''
    return sum of direct children nodes' degree (tag)
    '''
    if node.is_leaf():
        return str(int(node.tag))

    children = get_children_identifier(node, tree)
    child_deg = sum([int(tree[child].tag > 0) for child in children])

    return str(child_deg)

def map_all_child_deg(node, tree):
    '''
    return sum of all children nodes' degree (tag)
    '''
    if node.is_leaf():
        return str(int(node.tag))

    children = get_children_identifier(node, tree)
    child_deg = sum([int(tree[child].tag) for child in children])

    return str(child_deg)

def tree_to_bfs_string(tree, string_type='bfs'):
    bfs_node_list = [tree[node] for node in tree.expand_tree(mode=tree.WIDTH,
                                                             key=lambda x: (int(x.identifier.split('-')[0]), int(x.identifier.split('-')[1])))][1:]
    if string_type in ['bfs', 'group', 'bfs-tri', 'group-red', 'group-red-3']:
        bfs_value_list = [str(int(node.tag)) for node in bfs_node_list]
    elif string_type in ['bfs-deg', 'bfs-deg-group']:
        bfs_value_list = [map_child_deg(node, tree) for node in bfs_node_list]

    return ''.join(bfs_value_list)

def grouper(n, iterable, fillvalue=None):
    "grouper(3, 'ABCDEFG', 'x') --> ABC DEF Gxx"
    args = [iter(iterable)] * n
    return zip_longest(fillvalue=fillvalue, *args)

def bfs_string_to_tree(string, is_zinc=False, k=2):
    k_square = k**2
    tree = Tree()
    tree.create_node("root", "0")
    parent_node = tree["0"]
    node_deque = deque([])
    if is_zinc:
        node_groups = grouper_mol(string)
    else:
        node_groups = grouper(k_square, string)
    for node_1, node_2, node_3, node_4 in node_groups:
        parent_level = get_level(parent_node)
        cur_level_max = max([get_location(node) for node in tree.nodes.values() if get_level(node) == parent_level+1], default=0)
        for i, node_tag in enumerate([node_1, node_2, node_3, node_4], 1):
            if node_tag == None:
                break
            new_node = Node(tag=node_tag, identifier=f"{parent_level+1}-{cur_level_max+i}")
            tree.add_node(new_node, parent=parent_node)
            node_deque.append(new_node)
        parent_node = node_deque.popleft()
        while(parent_node.tag == '0'):
            if len(node_deque) == 0:
                return tree
            parent_node = node_deque.popleft()
    return tree

def clean_string(string):

    if "[pad]" in string:
        string = string[:string.index("[pad]")]

    return string

def train_val_test_split(
    data: list,
    data_name='GDSS_com',
    train_size: float = 0.7, val_size: float = 0.1, test_size: float = 0.2,
    seed: int = 42,
):
    if data_name in ['qm9', 'zinc']:
        # code adpated from https://github.com/harryjo97/GDSS
        with open(os.path.join(DATA_DIR, f'{data_name}/valid_idx_{data_name}.json')) as f:
            test_idx = json.load(f)
        if data_name == 'qm9':
            test_idx = test_idx['valid_idxs']
        test_idx = [int(i) for i in test_idx]
        train_idx = [i for i in range(len(data)) if i not in test_idx]
        test = [data[i] for i in test_idx]
        train_val = [data[i] for i in train_idx]
        train, val = train_test_split(train_val, train_size=train_size / (train_size + val_size), random_state=seed, shuffle=True)
    elif data_name in ['planar', 'sbm', 'proteins']:
        # code adapted from https://github.com/KarolisMart/SPECTRE
        test_len = int(round(len(data)*0.2))
        train_len = int(round((len(data) - test_len)*0.8))
        val_len = len(data) - train_len - test_len
        train, val, test = random_split(data, [train_len, val_len, test_len], generator=torch.Generator().manual_seed(1234))
    else:
        train_val, test = train_test_split(data, train_size=train_size + val_size, shuffle=False)
        train, val = train_test_split(train_val, train_size=train_size / (train_size + val_size), random_state=seed, shuffle=True)
    return train, val, test

def adj_to_graph(adj, is_cuda=False):
    if is_cuda:
        adj = adj.detach().cpu().numpy()
    G = nx.from_numpy_matrix(adj)
    G.remove_edges_from(nx.selfloop_edges(G))
    G.remove_nodes_from(list(nx.isolates(G)))
    if G.number_of_nodes() < 1:
        G.add_node(1)
    return G

def map_tree_pe(tree):
    depth = tree.depth()
    k = get_k(tree)
    size = int(depth*(k**2))
    pe = torch.zeros((size))
    for node in tree.nodes.values():
        node.data = pe
        if not node.is_root():
            parent = get_parent(node, tree)
            branch = get_children_identifier(parent, tree).index(node.identifier)
            current_pe = torch.zeros(int(k**2))
            current_pe[branch] = 1
            pe = torch.cat((current_pe, parent.data[:int(size-k**2)]))
            node.data = pe
    return tree

def map_new_ordered_graph(ordered_graph):
    '''
    Map ordered_graph object to ordered networkx graph
    '''
    org_graph = ordered_graph.graph
    ordering = ordered_graph.ordering
    mapping = {i: ordering.index(i) for i in range(len(ordering))}
    new_graph = nx.relabel_nodes(org_graph, mapping)
    return new_graph

# for redundant removed strings

def generate_final_tree_red(tree, k=2):
    tree_with_iden = add_zero_to_identifier(tree)
    final_tree = add_symmetry_to_tree(tree_with_iden, k)

    return final_tree

def generate_initial_tree_red(string_token_list, k=2):
    node_groups = [tuple(grouper_mol(string, k)[0]) for string in string_token_list]
    tree = Tree()
    tree.create_node("root", "0-0-0")
    parent_node = tree["0-0-0"]
    node_deque = deque([])
    for nodes in node_groups:
        parent_level = get_level(parent_node)
        cur_level_max = max([get_location(node) for node in tree.nodes.values() if get_level(node) == parent_level+1], default=0)
        for i, node_tag in enumerate(nodes, 1):
            if node_tag == None:
                break
            new_node = Node(tag=node_tag, identifier=f"{parent_level+1}-{cur_level_max+i}")
            tree.add_node(new_node, parent=parent_node)
            node_deque.append(new_node)
        parent_node = node_deque.popleft()
        while(parent_node.tag == '0'):
            if len(node_deque) == 0:
                return tree
            parent_node = node_deque.popleft()

    return tree

def find_new_identifier(node_id, index, is_dup=1):
    split = node_id.split('-')
    # k = 2
    if index == 1:
        num = 1
        num *= is_dup
        cur_pre_identifier = split[0] + '-' + split[1]
    # k = 3
    elif index == 2:
        num = 2
        num *= is_dup
        cur_pre_identifier = split[0] + '-' + split[1]
    elif index == 4:
        num = 1
        num *= is_dup
        cur_pre_identifier = split[0] + '-' + str(int(split[1])-2)
    elif index == 5:
        num = 1
        num *= is_dup
        cur_pre_identifier = split[0] + '-' + str(int(split[1])-1)
    else:
        num = 2
        num *= is_dup
        cur_pre_identifier = split[0] + '-' + split[1]

    new_last_identifier = int(split[2])-num
    return cur_pre_identifier + '-' + str(new_last_identifier)

def get_child_index(k):
    indices = []
    c = 0
    for i in range(k+1):
        for j in range(1,i+1):
            c=c+1
            if j != i:
                indices.append(c)
    return indices

def add_symmetry_to_tree(tree, k):
    k_square = k**2
    bfs_node_list = [tree[node] for node in tree.expand_tree(mode=tree.WIDTH, key=lambda x: (int(x.identifier.split('-')[0]), int(x.identifier.split('-')[1])))]
    node_list = [node for node in bfs_node_list[::-1] if not node.is_leaf()]
    for node in node_list:
        child_nodes = get_children_identifier(node, tree)
        if len(child_nodes) < k_square:
            postfixes = get_child_index(k)

            for index in postfixes:
                copy_node = tree.get_node(child_nodes[int(index)-1])
                new_node = Node(tag=copy_node.tag, identifier=find_new_identifier(copy_node.identifier, index))
                subtree = Tree(tree.subtree(child_nodes[int(index)-1]), deep=True)
                new_tree = Tree(subtree, deep=True)
                if len(subtree) > 1:
                    for nid, n in sorted(subtree.nodes.items(), key=lambda x: (int(x[0].split('-')[0]), int(x[0].split('-')[1]), int(x[0].split('-')[2]))):
                        count_dup = len([key for key in subtree.nodes.keys()
                                        if (key.split('-')[0] == nid.split('-')[0]) and (key.split('-')[1] == nid.split('-')[1])])
                        org_dup = len([key for key in tree.nodes.keys()
                                        if (key.split('-')[0] == nid.split('-')[0]) and (key.split('-')[1] == nid.split('-')[1])])
                        new_iden = find_new_identifier(nid, index, (count_dup+org_dup)*100)
                        while (new_iden in tree):
                            new = int(new_iden.split('-')[2]) - 1
                            new_iden = new_iden.split('-')[0] + '-' + new_iden.split('-')[1] + '-' +  str(new)
                        new_tree.update_node(nid, identifier=new_iden)
                    tree.paste(node.identifier, new_tree)

                else:
                    tree.add_node(new_node, parent=node)

    return tree

def add_zero_to_identifier(tree):
    new_tree = Tree(tree, deep=True)
    for node in tree.nodes:
        new_identifier = node
        while (len(new_identifier.split('-'))<3):
            new_identifier = new_identifier + '-10000'
        new_tree.update_node(node, identifier=new_identifier)
    return new_tree

def fix_symmetry(adj):
    sym_adj = torch.tril(adj) + torch.tril(adj).T
    return torch.where(sym_adj>0, 1, 0)

def map_deg_string(string):
    new_string = []
    group_queue = deque(grouper(4, string))
    group_queue.popleft()
    for index, char in enumerate(string):
        if len(group_queue) == 0:
            left = string[index:]
            break
        if char == '0':
            new_string.append(char)
        else:
            new_string.append(str(sum([int(char) for char in group_queue.popleft()])))

    return ''.join(new_string) + left

def remove_redundant(input_string, is_mol=False, k=2):
    k_square = k**2
    string = input_string[0:k_square]
    pos_list = list(range(1, k_square+1))
    str_pos_queue = deque([(s, p) for s, p in zip(string, pos_list)])
    if is_mol:
        group_list = list(grouper_mol(input_string))
    else:
        group_list = list(grouper(k_square, input_string))
    for cur_string in [''.join(token) for token in group_list][1:]:
        cur_parent, cur_parent_pos = str_pos_queue.popleft()
        # if value is 0, it cannot be parent node -> skip
        while((cur_parent == '0') and (len(str_pos_queue) > 0)):
            cur_parent, cur_parent_pos = str_pos_queue.popleft()
        # i: order of the child node in the same parent
        cur_pos = [cur_parent_pos*10+i for i in range(1,k_square+1)]
        # pos_list: final position of each node
        pos_list.extend(cur_pos)
        if is_mol:
            str_pos_queue.extend([(s, c) for s, c in zip(grouper_mol(cur_string)[0], cur_pos)])
        else:
            str_pos_queue.extend([(s, c) for s, c in zip(cur_string, cur_pos)])

    pos_list = [str(pos) for pos in pos_list]
    # prefix: diagonal
    prefixes = [str((i-1)*k + i) for i in range(1,k+1)]
    # posfix: upper diagonal
    postfixes = []
    for i in range(1,k+1):
        for j in range(i+1, k+1):
            postfixes.append(str((i-1)*k+j))

    # find positions ends with 2 including only 1 and 4
    remove_pos_prefix_list = [pos for i, pos in enumerate(pos_list)
                                if (pos[-1] in postfixes) and len((set(pos[:-1]))-set(prefixes))==0]
    remain_pos_index = [not pos.startswith(tuple(remove_pos_prefix_list)) for pos in pos_list]
    remain_pos_list = [pos for pos in pos_list if not pos.startswith(tuple(remove_pos_prefix_list))]

    # find cutting points (one block)
    cut_list = [i for i, pos in  enumerate(remain_pos_list) if pos[-1] == str(k**2)]
    cut_list_2 = [0]
    cut_list_2.extend(cut_list[:-1])
    cut_size_list = [i - j for i, j in zip(cut_list , cut_list_2)]
    cut_size_list[0] += 1
    if is_mol:
        final_string_list = list(compress([item for sublist in grouper_mol(input_string) for item in sublist], remain_pos_index))
    else:
        final_string_list = list(compress([*input_string], remain_pos_index))

    pos_list_iter = iter(final_string_list)
    final_string_cut_list = [list(islice(pos_list_iter, i)) for i in cut_size_list]

    return [''.join(l) for l in final_string_cut_list]

def get_max_len(data_name, order='C-M', k=2):
    total_strings = []
    k_square = k**2
    for split in ['train', 'test', 'val']:
        string_path = os.path.join(DATA_DIR, f"{data_name}/{order}/{data_name}_str_{split}_{k}.txt")
        strings = Path(string_path).read_text(encoding="utf=8").splitlines()

        total_strings.extend(strings)

    red_list = [remove_redundant(string, is_mol=False, k=k) for string in total_strings]
    # red_strings = [''.join(red) for red in red_list]

    max_len = max([len(string) for string in total_strings])
    group_max_len = max_len / k_square
    red_max_len = max([len(string) for string in red_list])

    return max_len, group_max_len, red_max_len


'''
2096. Step-By-Step Directions From a Binary Tree Node to Another
https://leetcode.com/problems/step-by-step-directions-from-a-binary-tree-node-to-another/
'''
from typing import List, Optional, Tuple

from test_tool import assert_value

# Definition for a binary tree node.
class TreeNode:
    def __init__(self, val=0, left=None, right=None):
        self.val = val
        self.left = left
        self.right = right

class Solution:
    def getDirections(self, root: Optional[TreeNode], startValue: int, destValue: int) -> str:
        def find(node, val):
            if not node:
                return False
            if node.val == val:
                res.extend(curr[:])
                return True

            curr.append(('L', node.val))
            if find(node.left, val):
                return True
            curr.pop()

            curr.append(('R', node.val))
            if find(node.right, val):
                return True
            curr.pop()
            return False

        res = start = []
        curr = []
        find(root, startValue)

        res = end = []
        curr.clear()
        find(root, destValue)

        idx = 0
        while idx < min(len(start), len(end)) and start[idx][0] == end[idx][0]:
            idx += 1
        start = ['U'] * (len(start) - idx)
        end = end[idx:]
        end = [source for source, v in end]
        start.extend(end)

        return ''.join(start)

    def _getDirections(self, root: Optional[TreeNode], startValue: int, destValue: int) -> str:
        cache_start = []
        self.find(root, startValue, cache_start)
        cache_dest = []
        self.find(root, destValue, cache_dest)
        i = 0
        cache_start = cache_start[::-1]
        cache_dest = cache_dest[::-1]
        while i < len(cache_start) and i < len(cache_dest) and cache_start[i][0] == cache_dest[i][0]:
            i += 1

        end = [s for v, s in cache_dest[i:]]
        start = 'U' * (len(cache_start) - i)

        return start + ''.join(end)

    def find(self, root: TreeNode, target: int, cache: List[Tuple[int, str]]) -> bool:
        if not root:
            return False
        if root.val == target:
            cache.append([root.val, None])
            return True
        left = self.find(root.left, target, cache)
        if left:
            cache[-1][1] = 'L'
            cache.append([root.val, None])
            return True
        right = self.find(root.right, target, cache)
        if right:
            cache[-1][1] = 'R'
            cache.append([root.val, None])
            return True
        return False


import unittest

from scipy.stats import nbinom

import openquake.hme.utils.stats as ss

p = [0.01, 0.2, 0.5]
q = [0.33, 0.22, 0.44]

kld = ss.kullback_leibler_divergence(p, q)

jsdiv = ss.jensen_shannon_divergence(p, q)

jsdist = ss.jensen_shannon_distance(p, q)

nb = nbinom(5, 0.5)

rvs = nb.rvs(1000)

nb_ps = ss.estimate_negative_binom_parameters(rvs)

print(nb_ps)


from io import BytesIO
from model import *
from reader import Reader
from storage import Segment
import datetime

def decode_key_type(kt):
    if kt == 0:
        return "group_metadata"
    elif kt == 1:
        return "offset_commit"
    elif kt == 2:
        return "noop"
    return "unknown"

def decode_member_proto(rdr):
    ret = {}
    ret['name'] = rdr.read_string()
    ret['metadata'] = rdr.read_iobuf().hex()
    return ret

def decode_member(rdr):
    ret = {}
    ret['member_id'] = rdr.read_string()
    ret['session_timeout'] = rdr.read_int64()
    ret['rebalance_timeout'] = rdr.read_int64()
    ret['instance_id'] = rdr.read_optional(lambda r: r.read_string())
    ret['protocol_type'] = rdr.read_string()
    ret['protocols'] = rdr.read_vector(decode_member_proto)
    ret['assignment'] = rdr.read_iobuf().hex()

    return ret

def decode_metadata(rdr):
    ret = {}
    ret['protocol_type'] = rdr.read_string()
    ret['generation_id'] = rdr.read_int32()
    ret['protocol_name'] = rdr.read_optional(lambda r: r.read_string())
    ret['leader'] = rdr.read_optional(lambda r: r.read_string())
    ret['state_timestamp'] = rdr.read_int32()
    ret['members'] = rdr.read_vector(decode_member)
    return ret

def decode_metadata_key(key_rdr):
    ret = {}
    key_buf = key_rdr.read_iobuf()
    rdr = Reader(BytesIO(key_buf))
    ret['id'] = rdr.read_string()
    return ret

def decode_offset_commit_key(key_rdr):
    ret = {}
    key_buf = key_rdr.read_iobuf()
    rdr = Reader(BytesIO(key_buf))
    ret['id'] = rdr.read_string()
    ret['topic'] = rdr.read_string()
    ret['partition'] = rdr.read_int32()
    return ret

def decode_offset_commit(v_rdr):
    ret = {}
    ret['committed_offset'] = v_rdr.read_int64()
    ret['leader_epoch'] = v_rdr.read_int32()
    ret['committed_metadata'] = v_rdr.read_optional(lambda r: r.read_string())
    return ret

def decode_record(hdr, r):
    v = {}
    v['epoch'] = hdr.first_ts
    v['offset'] = hdr.base_offset + r.offset_delta
    v['ts'] = datetime.datetime.utcfromtimestamp(
        hdr.first_ts / 1000.0).strftime('%Y-%m-%d %H:%M:%S')
    k_rdr = Reader(BytesIO(r.key))

    v['type'] = decode_key_type(k_rdr.read_int8())

    if v['type'] == "group_metadata":
        v["key"] = decode_metadata_key(k_rdr)
        if r.value:
            rdr = Reader(BytesIO(r.value))
            v['value'] = decode_metadata(rdr)
        else:
            v['value'] = 'tombstone'
    elif v['type'] == "offset_commit":
        v["key"] = decode_offset_commit_key(k_rdr)
        if r.value:
            rdr = Reader(BytesIO(r.value))
            v['value'] = decode_offset_commit(rdr)
        else:
            v['value'] = 'tombstone'

    return v

class GroupsLog:
    def __init__(self, ntp):
        self.ntp = ntp
        self.records = []

    def decode(self):
        paths = []
        for path in self.ntp.segments:
            paths.append(path)
        paths.sort()
        for path in paths:
            s = Segment(path)
            for b in s:
                if b.header.type != 1:
                    continue
                for r in b:
                    self.records.append(decode_record(b.header, r))


#!/usr/bin/env python3

from argparse import ArgumentParser
from big_o import big_o
import numpy as np
from tqdm import trange
from algos import cp, wd, opt1, feas, opt2
from generators import gen_random_circuit
from utils import load_graph, save_graph

def check_time_complexity():
    MIN_N = 10
    MAX_N = 100
    N_MEASURES = 46
    N_TIMINGS = 3

    def gen_random_circuit_nodes(n):
        n_edges = round(n / 8 * 11)
        # print(f'Generating circuit with {n} vertices and {n_edges} edges...')
        return gen_random_circuit(n, n_edges)

    def gen_random_circuit_edges(n):
        n_nodes = round(n / 11 * 8)
        # print(f'Generating circuit with {n_nodes} vertices and {n} edges...')
        return gen_random_circuit(n_nodes, n)

    print('Algorithm CP  -  Expected O(E)')
    best, _ = big_o(cp, gen_random_circuit_edges,
                    min_n=MIN_N, max_n=MAX_N,
                    n_measures=N_MEASURES, n_timings=N_TIMINGS)
    print(best, end='\n\n')

    print('Algorithm WD  -  Expected O(V^3)')
    best, _ = big_o(wd, gen_random_circuit_nodes,
                    min_n=MIN_N, max_n=MAX_N,
                    n_measures=N_MEASURES, n_timings=N_TIMINGS)
    print(best, end='\n\n')

    print('Algorithm OPT1  -  Expected O(V^3 log V)')
    best, _ = big_o(opt1, gen_random_circuit_nodes,
                    min_n=MIN_N, max_n=MAX_N,
                    n_measures=N_MEASURES, n_timings=N_TIMINGS)
    print(best, end='\n\n')

    print('Algorithm FEAS  -  Expected O(VE)')
    best, _ = big_o(lambda g: feas(g, np.inf), gen_random_circuit_nodes,
                    min_n=MIN_N, max_n=MAX_N,
                    n_measures=N_MEASURES, n_timings=N_TIMINGS)
    print('NODES:', best)
    best, _ = big_o(lambda g: feas(g, np.inf), gen_random_circuit_edges,
                    min_n=MIN_N, max_n=MAX_N,
                    n_measures=N_MEASURES, n_timings=N_TIMINGS)
    print('EDGES:', best, end='\n\n')

    print('Algorithm OPT2  -  Expected O(VE log V)')
    best, _ = big_o(opt2, gen_random_circuit_nodes,
                    min_n=MIN_N, max_n=MAX_N,
                    n_measures=N_MEASURES, n_timings=N_TIMINGS)
    print(best, end='\n\n')

def random_test(n=10000):
    for _ in trange(n):
        V = np.random.randint(5, 30)
        E = np.random.randint(5, 30)
        g = gen_random_circuit(V, E)
        cpg = cp(g)
        cpr1 = cp(opt1(g))
        cpr2 = cp(opt2(g))
        assert cpr1 == cpr2 and cpr1 <= cpg and cpr2 <= cpg

def run(g, save=None, show_wd=False):
    cpg = cp(g)
    print(f'The original graph has a clock period of {cpg}')
    print('Running algorithm OPT1')
    g1 = opt1(g, show_wd=show_wd)
    if save is not None:
        path = save+'_opt1.dot'
        save_graph(g1, path)
        print(f'Output graph saved to {path}')
    cpr1 = cp(g1)
    print(f'The graph returned by OPT1 has a clock period of {cpr1}')
    print('Running algorithm OPT2')
    g2 = opt2(g, show_wd=show_wd)
    if save is not None:
        path = save+'_opt2.dot'
        save_graph(g2, path)
        print(f'Output graph saved to {path}')
    cpr2 = cp(g2)
    print(f'The graph returned by OPT2 has a clock period of {cpr2}')

if __name__ == '__main__':
    parser = ArgumentParser()
    parser.add_argument('--output', '-o', help='File where to save the output graphs (please omit the extension, it '
                                               'will be added automatically')
    parser.add_argument('--show-wd', action='store_true', help='Show matrices W and D')
    subparsers = parser.add_subparsers()
    parser_random = subparsers.add_parser('random', help='Run the algorithms on a random graph')
    parser_random.add_argument('--nodes', '-n', type=int, default=8, help='The number of nodes (default 8)')
    parser_random.add_argument('--edges', '-e', type=int, default=11, help='The number of edges (default 11)')
    parser_file = subparsers.add_parser('file', help='Run the algorithms on a graph loaded from a provided DOT file')
    parser_file.add_argument('file', help='The DOT file from which the graph is loaded')
    args = parser.parse_args()
    if 'nodes' in args and 'edges' in args:
        print(f'Generating random graph with {args.nodes} nodes and {args.edges} edges')
        g = gen_random_circuit(args.nodes, args.edges)
        run(g, save=args.output, show_wd=args.show_wd)
    elif 'file' in args:
        print(f'Loading graph from {args.file}')
        g = load_graph(args.file)
        run(g, save=args.output, show_wd=args.show_wd)
    else:
        print('ERROR: unrecognized argument')


import base64
print("Enter the encoding to be used")
print("1 for base64")
print("2 for utf-16")
print("3 for utf-32")
en=int(input())
str_input=input("Enter the string to encode")

if en==1:
    str_base64_en = base64.b64encode(str_input.encode())
    print("Encoded string",str_base64_en)
elif en==2:
    str_utf_8=str_input.encode('utf-16','strict')
    print("Encoded string",str_utf_8)

elif en==3:
    str_utf_32=str_input.encode('utf-32','strict')
    print("Encoded string",str_utf_32)

if en==1:
    str_base64_de=base64.b64decode(str_base64_en)
    print("Decoded string",str_base64_de)
elif en==2:
    str_utf_8_de=str_utf_8.decode('utf-16','strict')
    print("Decode string",str_utf_8_de)
elif en==3:
    str_utf_32_de=str_utf_32.decode('utf-32','strict')
    print("Decode string",str_utf_32_de)


###################################
# File Name : async_with.py (Python 3.5 or later)
###################################
#!/usr/bin/python3

import asyncio

class AsynchronousReader:

    def __init__(self, file_name):
        self.file_name = file_name
        self.file = None

    async def __aenter__(self):
        try:
            self.file = open(self.file_name, "rb")
        except:
            print ("File open error.")
            raise Exception
        else:
            return self.file

    async def __aexit__(self, exc_type, exc_value, traceback):
        self.file.close()

async def read_file(file_name):
    async with AsynchronousReader(file_name) as af:
        for line in af.readlines():
            print (line.decode("utf-8").strip())

if __name__ == "__main__":
    loop = asyncio.get_event_loop()
    try:
        loop.run_until_complete(read_file("./python_foudation_mission_statement"))
    finally:
        loop.close()


from django import template

register = template.Library()

@register.filter(name='num_suffix')
def num_suffix(number):
    if number == 1:
        return '第1名'
    if number == 2:
        return '第2名'
    if number == 3:
        return '第3名'
    if 3 < number <= 12:
        return '第%s' % number


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations
from django.conf import settings

class Migration(migrations.Migration):

    dependencies = [
        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
        ('rtiapp', '0001_initial'),
    ]

    operations = [
        migrations.CreateModel(
            name='Follow_department',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('entry_date', models.DateTimeField(auto_now_add=True)),
                ('followee', models.ForeignKey(related_name='department_followee', to='rtiapp.Department')),
                ('follower', models.ForeignKey(related_name='department_follower', to=settings.AUTH_USER_MODEL)),
            ],
            options={
            },
            bases=(models.Model,),
        ),
        migrations.CreateModel(
            name='Follow_state',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('entry_date', models.DateTimeField(auto_now_add=True)),
                ('followee', models.ForeignKey(related_name='state_followee', to='rtiapp.State')),
                ('follower', models.ForeignKey(related_name='state_follower', to=settings.AUTH_USER_MODEL)),
            ],
            options={
            },
            bases=(models.Model,),
        ),
        migrations.CreateModel(
            name='Follow_topic',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('entry_date', models.DateTimeField(auto_now_add=True)),
                ('followee', models.ForeignKey(related_name='topic_followee', to='rtiapp.Tag')),
                ('follower', models.ForeignKey(related_name='topic_follower', to=settings.AUTH_USER_MODEL)),
            ],
            options={
            },
            bases=(models.Model,),
        ),
        migrations.AlterField(
            model_name='follow_user',
            name='followee',
            field=models.ForeignKey(related_name='user_followee', to=settings.AUTH_USER_MODEL),
            preserve_default=True,
        ),
        migrations.AlterField(
            model_name='follow_user',
            name='follower',
            field=models.ForeignKey(related_name='user_follower', to=settings.AUTH_USER_MODEL),
            preserve_default=True,
        ),
    ]


from pathlib import Path
from typing import Optional

from tap import Tap

class CLIOptions(Tap):
    correlation_file: Path  # correlation rules to use
    logfile: Path  # (possibly gzipped) log file
    extra: bool = False  # show extra log lines (not matched by iso_regex)
    summary: bool = False  # show summary
    nolog: bool = False  # don't show log
    max: bool = False  # show max durations
    begin: Optional[str] = None  # start from a date
    end: Optional[str] = None  # stop to a date

    def configure(self) -> None:
        self.add_argument("correlation_file", metavar="TOML_FILE", type=str)
        self.add_argument("logfile", metavar="LOG_FILE", type=str)
        self.add_argument(
            "--extra",
        )
        self.add_argument(
            "--summary",
        )
        self.add_argument(
            "--nolog",
        )
        self.add_argument("-b", "--begin", metavar="DATE", type=str)
        self.add_argument("-e", "--end", metavar="DATE", type=str)


import datetime, pytz, re, helper

def grab(channel, timespan):
	tz = pytz.timezone("Europe/Berlin")
	now = datetime.datetime.now(tz)
	shows = []
	a = 0
	if now.time().hour < 7:
		a = -1
	for i in range(a, 14):
		date = now + datetime.timedelta(days=i)
		datestring = "%s.%s.%s" % (date.day, date.month, date.year)
		text = helper.download("http://programm.ard.de/TV/Programm/Sender?datum=" + date.strftime("%d.%m.%Y") + "&hour=0&sender=" + channel)
		if text is None:
			continue

		sections = helper.split(text, "<li class=\"eid", "
")
		laststart = datetime.datetime.min.replace(tzinfo=tz)
		for section in sections:
			show = {}
			temp = re.search("<span class=\"date[\s\S]*?(\d\d):(\d\d)", section)
			show["start"] = date.replace(hour=int(temp.group(1)), minute=int(temp.group(2)), second=0, microsecond=0)
			if show["start"] < laststart:
				date += datetime.timedelta(days=1)
				show["start"] += datetime.timedelta(days=1)

			if (show["start"] - now).total_seconds() / 3600 > timespan:
				lastshow = True
			else:
				lastshow = False

			laststart = show["start"]

			show["title"] = helper.cleanup(re.search("\s*([^<]*?)[\t\n]", section).group(1))
			temp = re.search("\s*([^<]*?)[\t\n]", section)
			if temp is not None:
				subtitle = temp.group(1)
				if subtitle:
					show["sub-title"] = helper.cleanup(subtitle)

			temp = re.search("<a class=\"sendungslink[\s\S]*?href=\"(.*?)\"", section)
			if temp is not None:
				show["details-url"] = "http://programm.ard.de" + temp.group(1)
			shows.append(show)
			if lastshow:
				return shows
	return shows

def grabdetails(url):
	text = helper.download(url)
	if text is None:
		return None
	show = {}
	description = helper.cut(text, "<meta name=\"description\" content=\"", "\" />")
	if description is not None:
		show["desc"] = helper.cleanup(description)
	return show

def channellist():
	text = helper.download("http://programm.ard.de/")
	channels = helper.split(text, "Tagesprogramm::", "")
	result = []
	for channel in channels:
		temp = re.search("Tagesprogramm::(.*?)\".*\?sender\=-?(.*?)\&", channel)
		result.append((temp.group(2), temp.group(1), temp.group(1)))
	return result


from django.db import models
from django.utils.text import slugify

from wagtail.core.models import Page
from wagtail.admin.edit_handlers import FieldPanel, MultiFieldPanel
from wagtail.images.edit_handlers import ImageChooserPanel
from wagtailmarkdown.edit_handlers import MarkdownPanel
from wagtailmarkdown.fields import MarkdownField

from wagtail_graphql.models import GraphQLEnabledModel, GraphQLField

# Create your models here.

class CharacterPage(GraphQLEnabledModel, Page):
    """A page of character list."""
    nickname = models.CharField(u"称号", max_length=15, null=True)
    name = models.CharField(u"名前", max_length=35, null=True)
    character_id = models.CharField(u"キャラクターID", max_length=6, null=True)
    image = models.ForeignKey(
        'wagtailimages.Image',
        null=True,
        blank=True,
        on_delete=models.SET_NULL,
        related_name='+',
        verbose_name=u'画像',
    )
    description = models.CharField(u'概要', max_length=255, null=True)
    introduction = MarkdownField(verbose_name=u"説明", null=True)
    game_name = models.CharField(u"登録されているゲーム", max_length=20, null=True)
    character_page_url = models.CharField(u"キャラクターのページ", max_length=255, null=True)

    content_panels = Page.content_panels + [
        FieldPanel('nickname'),
        FieldPanel('name'),
        FieldPanel('character_id'),
        ImageChooserPanel('image'),
        FieldPanel('description'),
        MarkdownPanel('introduction'),
        FieldPanel('game_name'),
        FieldPanel('character_page_url'),
    ]

    promote_panels = [
        MultiFieldPanel(Page.promote_panels, "Common page configuration"),
    ]

    graphql_fields = [
        GraphQLField("nickname"),
        GraphQLField("name"),
        GraphQLField("character_id"),
        GraphQLField("image"),
        GraphQLField("description"),
        GraphQLField("introduction"),
        GraphQLField("game_name"),
        GraphQLField("character_page_url"),
    ]

    def clean(self):
        super().clean()
        new_title = '%s(%s)' % (self.name, self.character_id)
        new_slug = '%s' % self.character_id
        self.title = new_title
        self.slug = slugify(new_slug)

class CharacterIndexPage(GraphQLEnabledModel, Page):
    intro = MarkdownField(null=True)

    def child_pages(self):
        return CharacterPage.objects.live().child_of(self)

    content_panels = Page.content_panels + [
        MarkdownPanel('intro', classname='full')
    ]

    graphql_fields = [
        GraphQLField('intro'),
    ]

    subpage_types = ['CharacterPage']


#!/usr/bin/env python3

from datetime import datetime, timezone
import json
import os
import pathlib
import socket
import subprocess

def get_commit_id():
    return (
        subprocess.run(
            ["git", "describe", "--always", "--dirty"],
            capture_output=True,
        )
        .stdout.decode()
        .strip()
    )

def get_basic_metadata(ensembles_filename):
    metadata = {}
    ensembles_file = pathlib.Path(ensembles_filename)

    metadata[
        "_comment"
    ] = "This file and all the files in this directory were generated automatically. Do not modify them; re-run the analysis workflow!"
    metadata["workflow_run"] = {
        "completed": datetime.now(timezone.utc).isoformat(),
        "user_name": os.getlogin(),
        "machine_name": socket.gethostname(),
    }
    metadata["ensembles_metadata"] = {
        "last_updated": datetime.fromtimestamp(
            ensembles_file.stat().st_mtime, timezone.utc
        ).isoformat()
    }
    metadata["analysis_code"] = {"version": get_commit_id()}

    return metadata

def stamp_provenance(ensembles_filename):
    metadata = get_basic_metadata(ensembles_filename)
    for dirname in "final_plots", "final_tables":
        with open(pathlib.Path(dirname) / "info.json", "w") as info_file:
            info_file.write(json.dumps(metadata, sort_keys=True, indent=4))