''' Agents: stop/random/shortest/seq2seq  '''

import json
import os
import sys
import numpy as np
import random
import time
import math

import torch
import torch.nn as nn
import torch.distributions as D
from torch.autograd import Variable
from torch import optim
import torch.nn.functional as F

import utils
from env import R2RBatch
from utils import padding_idx
from param import args
from collections import defaultdict

import scipy.stats

class BaseAgent(object):
    ''' Base class for an R2R agent to generate and save trajectories. '''

    def __init__(self, env, results_path):
        self.env = env
        self.results_path = results_path
        random.seed(args.seed)
        self.results = {}
        self.losses = [] # For learning agents
        self.IL_losses = []
        self.WeTA_losses = []
        self.RL_losses = []
        self.WaTA_losses = []

    def write_results(self):
        output = [{'inst_idx': k, 'trajectory': v} for k, v in self.results.iteritems()]
        with open(self.results_path, 'w') as f:
            json.dump(output, f)

    def rollout(self):
        ''' Return a list of dicts containing instr_id:'xx', path:[(viewpointId, heading_rad, elevation_rad)]  '''
        raise NotImplementedError

    @staticmethod
    def get_agent(name):
        return globals()[name+"Agent"]

    def test(self):
        self.env.reset_epoch()
        self.losses = []
        self.WeTA_losses = []
        self.RL_losses = []
        self.WaTA_losses = []
        self.IL_losses = []

        self.results = {}
        # We rely on env showing the entire batch before repeating anything
        #print 'Testing %s' % self.__class__.__name__
        looped = False
        while True:
            for traj in self.rollout():
                if traj['inst_idx'] in self.results:
                    looped = True
                else:
                    self.results[traj['inst_idx']] = traj['path']
            if looped:
                break

class AskOracle(object):
    def __init__(self):
        self.uncertain_threshold = args.uncertain_threshold
        self.DONT_ASK = 0
        self.ASK = 1

    def isUncertain(self, agent_dist):
        uniform = [1. / len(agent_dist)] * len(agent_dist)
        entropy_gap = scipy.stats.entropy(uniform) - scipy.stats.entropy(agent_dist)
        if entropy_gap < self.uncertain_threshold - 1e-9:
            return self.ASK, 'uncertain '
        else:
            return self.DONT_ASK, ''

    def __call__(self, agent_dist, last_distance, cur_distance):
        finalreason = ""
        ask_1, ask_reason_1 = self.isUncertain(agent_dist)
        if ask_1 == 1:
            finalreason += ask_reason_1

        if last_distance - cur_distance <= 0: # no progress
            ask_2 = 1
            ask_reason_2 = 'no progress'
        else:
            ask_2 = 0
            ask_reason_2 = ''

        if ask_2 == 1:
            finalreason += ask_reason_2

        finalask = 1 if (ask_1 or ask_2) else 0

        return finalask, finalreason

class SCoA_Agent(BaseAgent):
    ''' An agent based on an LSTM seq2seq model with attention. '''

    # For now, the agent can't pick which forward move to make - just the one in the middle
    model_actions = ['left', 'right', 'up', 'down', 'forward', '<end>', '<start>', '<ignore>']
    env_actions = {
      "left": ([0],[-1], [0]), # left
      "right": ([0], [1], [0]), # right
      "up": ([0], [0], [1]), # up
      "down": ([0], [0],[-1]), # down
      "forward": ([1], [0], [0]), # forward
      "<end>": ([0], [0], [0]), # <end>
      "<start>": ([0], [0], [0]), # <start>
      "<ignore>": ([0], [0], [0])  # <ignore>
    }
    feedback_options = ['teacher', 'argmax', 'sample']

    def __init__(self, env, results_path, encoder, decoder, critic, WeTA, episode_len=20, visable=False, path_type='planner_path'):
        super(SCoA_Agent, self).__init__(env, results_path)
        self.feature_size = 2048
        self.encoder = encoder
        self.decoder = decoder

        self.critic = critic
        self.WeTA = WeTA

        self.episode_len = episode_len
        self.losses = []
        self.IL_losses = []
        self.criterion = nn.CrossEntropyLoss(ignore_index=args.ignoreid, size_average=False)

        self.WeTA_criterion = nn.CrossEntropyLoss(size_average=False)
        self.WeTA_losses = []

        self.RL_losses = []
        self.WaTA_losses = []

        self.logs = defaultdict(list)

    @staticmethod
    def n_inputs():
        return len(SCoA_Agent.model_actions)

    @staticmethod
    def n_outputs():
        return len(SCoA_Agent.model_actions)-2 # Model doesn't output start or ignore

    # prepare candidate length and feature(bs, max_candidate_leng, 2048+4)
    def _candidate_variable(self, obs):
        candidate_leng = [len(ob['candidate']) + 1 for ob in obs]       # +1 is for the end
        candidate_feat = np.zeros((len(obs), max(candidate_leng), self.feature_size + args.angle_feat_size), dtype=np.float32)
        # Note: The candidate_feat at len(ob['candidate']) is the feature for the END
        # which is zero in my implementation
        for i, ob in enumerate(obs):
            for j, c in enumerate(ob['candidate']):
                candidate_feat[i, j, :] = c['feature']                         # Image feat
        return torch.from_numpy(candidate_feat).cuda(), candidate_leng

    # get angle feature, image feature, candidate feature, candidate leng
    def get_input_feat(self, obs):
        input_a_t = np.zeros((len(obs), args.angle_feat_size), np.float32)
        for i, ob in enumerate(obs):
            input_a_t[i] = utils.angle_feature(ob['heading'], ob['elevation'])
        input_a_t = torch.from_numpy(input_a_t).cuda()

        f_t = self._feature_variable(obs)      # Image features from obs
        candidate_feat, candidate_leng = self._candidate_variable(obs)

        # input_a_t: shape(bs, angle_feat_size)
        # f_t: shape(bs, 36, 2048)
        # candidate_feat: shape(bs, max_candidate_leng, 2048+4)
        # candidate_leng: shape(bs, 1)
        return input_a_t, f_t, candidate_feat, candidate_leng

    def _feature_variable(self, obs):
        ''' Extract precomputed features into variable. '''
        features = np.empty((len(obs), args.num_view, self.feature_size + args.angle_feat_size), dtype=np.float32)
        # feature_size = obs[0]['feature'].shape[0]
        # features = np.empty((len(obs), self.feature_size), dtype=np.float32)
        for i,ob in enumerate(obs):
            features[i,:] = ob['feature']
        return Variable(torch.from_numpy(features), requires_grad=False).cuda()

    def get_nav_feature_variable(self, obs):
        features = np.empty((len(obs), 5, args.num_view, self.feature_size), dtype=np.float32)
        for i, ob in enumerate(obs):
            features[i, :] = ob['nav_history_feature']
        return Variable(torch.from_numpy(features), requires_grad=False).cuda()

    def _teacher_action(self, obs, ended):
        """
        Extract teacher actions into variable.
        :param obs: The observation.
        :param ended: Whether the action seq is ended
        :return:
        """
        a = np.zeros(len(obs), dtype=np.int64)
        for i, ob in enumerate(obs):
            if ended[i]:                                            # Just ignore this index
                a[i] = args.ignoreid
            else:
                for k, candidate in enumerate(ob['candidate']):
                    if candidate['viewpointId'] == ob['teacher']:   # Next view point
                        a[i] = k
                        break
                else:   # Stop here
                    try:
                        assert ob['teacher'] == ob['viewpoint']         # The teacher action should be "STAY HERE"
                    except:
                        import pdb; pdb.set_trace()
                    a[i] = len(ob['candidate'])
        return torch.from_numpy(a).cuda()

    def make_equiv_action(self, a_t, perm_obs, perm_idx=None, traj=None):
        """
        Interface between Panoramic view and Egocentric view
        It will convert the action panoramic view action a_t to equivalent egocentric view actions for the simulator
        """
        def take_action(i, idx, name):
            if type(name) is int:       # Go to the next view
                self.env.env.sims[idx].makeAction([name], [0], [0])
            else:                       # Adjust
                self.env.env.sims[idx].makeAction(*self.env_actions[name])
            state = self.env.env.sims[idx].getState()[0]
            if traj is not None:
                traj[i]['path'].append((state.location.viewpointId, state.heading, state.elevation))

        if perm_idx is None:
            perm_idx = range(len(perm_obs))
        for i, idx in enumerate(perm_idx):
            action = a_t[i]
            if action != -1:            # -1 is the <stop> action
                select_candidate = perm_obs[i]['candidate'][action]
                src_point = perm_obs[i]['viewIndex']
                trg_point = select_candidate['pointId']
                src_level = (src_point ) // 12   # The point idx started from 0
                trg_level = (trg_point ) // 12
                while src_level < trg_level:    # Tune up
                    take_action(i, idx, 'up')
                    src_level += 1
                while src_level > trg_level:    # Tune down
                    take_action(i, idx, 'down')
                    src_level -= 1
                while self.env.env.sims[idx].getState()[0].viewIndex != trg_point:    # Turn right until the target
                    take_action(i, idx, 'right')
                assert select_candidate['viewpointId'] == \
                       self.env.env.sims[idx].getState()[0].navigableLocations[select_candidate['idx']].viewpointId
                take_action(i, idx, select_candidate['idx'])

    def update_ctx(self, ctx, h_t, c_t, ans_ctx, ans_h_t, ans_c_t, predict_WeTA):
        predict_WeTA = predict_WeTA.unsqueeze(-1) # (bs, 1)

        ctx = ctx + predict_WeTA * ans_ctx
        h_t = h_t + predict_WeTA * ans_h_t
        c_t = c_t + predict_WeTA * ans_c_t

        return ctx, h_t, c_t

    def getangle_feature(self, obs):
        ''' Extract precomputed features into variable. '''
        features = np.empty((len(obs), self.feature_size + args.angle_feat_size), dtype=np.float32)
        for i,ob in enumerate(obs):
            features[i,:] = ob['feature'][ob['viewIndex']]
        return Variable(torch.from_numpy(features), requires_grad=False).cuda()

    def rollout(self, iter=None, train_RL=True):

        obs = np.array(self.env.reset())
        batch_size = len(obs)

        # Record starting point
        traj = [{
            'inst_idx': ob['inst_idx'],
            'path': [(ob['viewpoint'], ob['heading'], ob['elevation'])]
        } for ob in obs]

        # Init the reward shaping
        last_dist = np.zeros(batch_size, np.float32)
        for i, ob in enumerate(obs):   # The init distance from the view point to the target
            last_dist[i] = ob['distance']

        # ctx: shape(bs, 1, 512)
        # h_t: shape(bs, 512)
        # c_t: shape(bs, 512)
        # tar_enc: shape(bs, 1, 512)
        ctx, h_t, c_t, tar_enc = self.encoder(obs, cur_img=None, next_img=None, tar_enc=None, encoder_target=True)

        # Initialization the tracking state
        ended = np.array([False] * batch_size) # Indices match permuation of the model, not env

        # For test result submission
        visited = [set() for _ in obs]

        # Init the logs
        rewards = []
        hidden_states = []
        policy_log_probs = []
        masks = []
        entropys = []

        # Do a sequence rollout and calculate the loss
        self.loss = 0
        self.IL_loss = 0
        self.WeTA_loss = 0
        self.RL_loss = 0
        self.WaTA_loss = 0

        h1 = h_t
        for t in range(self.episode_len):
            input_a_t, f_t, candidate_feat, candidate_leng = self.get_input_feat(obs)

            WeTA_logit, predict_WeTA = self.WeTA(h_t)  # (bs, )

            cur_imgfeature = self._feature_variable(obs)

            next_vp_imgfeature = np.array([ob['next_vp_imgfeature'] for ob in obs])
            next_vp_imgfeature = Variable(torch.from_numpy(next_vp_imgfeature), requires_grad=False).cuda()

            WaTA_loss, ans_ctx, ans_h_t, ans_c_t = \
                    self.encoder(obs=obs, cur_img=cur_imgfeature, next_img=next_vp_imgfeature, tar_enc=tar_enc)
            # udpate ctx, h_t, c_t
            ctx, h_t, c_t = self.update_ctx(ctx, h_t, c_t, ans_ctx, ans_h_t, ans_c_t, predict_WeTA)

            # add env dropout
            noise = self.decoder.drop_env(torch.ones(self.feature_size).cuda())
            candidate_feat[..., :-args.angle_feat_size] *= noise
            f_t[..., :-args.angle_feat_size] *= noise

            h_t, c_t, logit, h1 = self.decoder(input_a_t, f_t, candidate_feat,
                                    h_t, h1, c_t, ctx, None)

            hidden_states.append(h_t)
            candidate_mask = utils.length2mask(candidate_leng)

            # if 'test' in self.env.splits:
            if args.submit == 'True':     # Avoding cyclic path
                for ob_id, ob in enumerate(obs):
                    visited[ob_id].add(ob['viewpoint'])
                    for c_id, c in enumerate(ob['candidate']):
                        if c['viewpointId'] in visited[ob_id]:
                            candidate_mask[ob_id][c_id] = 1
            logit.masked_fill_(candidate_mask, -float('inf'))

            # Supervised training
            if 'test' not in self.env.splits:
                self.loss += WaTA_loss
                self.WaTA_loss += WaTA_loss

                # WeTA
                WeTA_target = []
                for ob_id, ob in enumerate(obs):
                    if ended[ob_id]:
                        WeTA_target.append(AskOracle().DONT_ASK)
                    else:
                        WeTA_t, _ = AskOracle()(logit[ob_id].cpu().detach().numpy(), last_dist[ob_id], ob['distance'])
                        WeTA_target.append(WeTA_t)
                WeTA_target = Variable(torch.Tensor(WeTA_target).long(), requires_grad=False).cuda()

                WeTA_loss = self.WeTA_criterion(WeTA_logit, WeTA_target)
                self.loss += WeTA_loss
                self.WeTA_loss += WeTA_loss

                target = self._teacher_action(obs, ended)

                IL_loss = self.criterion(logit, target)
                if not math.isinf(IL_loss):
                    self.loss += IL_loss
                    self.IL_loss += IL_loss

            probs = F.softmax(logit, dim=1)
            c = D.Categorical(probs)
            # RL
            self.logs['entropy'].append(c.entropy().sum().item())  # For log
            entropys.append(c.entropy())  # For optimization
            a_t = c.sample().detach()
            policy_log_probs.append(c.log_prob(a_t))

            cpu_a_t = a_t.cpu().numpy()
            for i, next_id in enumerate(cpu_a_t):
                if next_id == (candidate_leng[i]-1) or next_id == args.ignoreid:    # The last action is <end>
                    cpu_a_t[i] = -1

            self.make_equiv_action(cpu_a_t, obs, None, traj)
            obs = np.array(self.env._get_obs())

            dist = np.zeros(batch_size, np.float32)
            reward = np.zeros(batch_size, np.float32)
            mask = np.ones(batch_size, np.float32)
            for i, ob in enumerate(obs):
                dist[i] = ob['distance']
                if ended[i]:            # If the action is already finished BEFORE THIS ACTION.
                    reward[i] = 0.
                    mask[i] = 0.
                else:       # Calculate the reward
                    if predict_WeTA[i] >= 0.5:
                        reward[i] = args.ask_reward
                    else:
                        reward[i] = 0
                    action_idx = cpu_a_t[i]
                    if action_idx == -1:        # If the action now is end
                        if dist[i] < 3:         # Correct
                            reward[i] = reward[i] + 3.
                        else:                   # Incorrect
                            reward[i] = reward[i] - 3.
                    else:                       # The action is not end
                        pro = - (dist[i] - last_dist[i])      # Change of distance
                        if pro > 0:                           # Quantification
                            reward[i] = reward[i] + 2
                        elif pro <= 0:
                            reward[i] = reward[i] - 2

            rewards.append(reward)
            masks.append(mask)
            last_dist[:] = dist

            ended[:] = np.logical_or(ended, (cpu_a_t == -1))

            # Early exit if all ended
            if ended.all():
                break

            torch.cuda.empty_cache()

        torch.cuda.empty_cache()
        # Last action in A2C
        input_a_t, f_t, candidate_feat, candidate_leng = self.get_input_feat(obs)
        candidate_feat[..., :-args.angle_feat_size] *= noise
        f_t[..., :-args.angle_feat_size] *= noise

        last_h_, _, _, _ = self.decoder(input_a_t, f_t, candidate_feat,
                         h_t, h1, c_t, ctx, None)
        RL_loss = 0.

        # NOW, A2C!!!
        # Calculate the final discounted reward
        last_value__ = self.critic(last_h_).detach()    # The value esti of the last state, remove the grad for safety
        discount_reward = np.zeros(batch_size, np.float32)  # The inital reward is zero
        for i in range(batch_size):
            if not ended[i]:        # If the action is not ended, use the value function as the last reward
                discount_reward[i] = last_value__[i]

        length = len(rewards)
        total = 0
        for t in range(length-1, -1, -1):
            discount_reward = discount_reward * args.gamma + rewards[t]   # If it ended, the reward will be 0
            mask_ = Variable(torch.from_numpy(masks[t]), requires_grad=False).cuda()
            clip_reward = discount_reward.copy()
            r_ = Variable(torch.from_numpy(clip_reward), requires_grad=False).cuda()
            v_ = self.critic(hidden_states[t])
            a_ = (r_ - v_).detach()

            # r_: The higher, the better. -ln(p(action)) * (discount_reward - value)
            RL_loss += (-policy_log_probs[t] * a_ * mask_).sum()
            RL_loss += (((r_ - v_) ** 2) * mask_).sum() * 0.5     # 1/2 L2 loss
            if self.feedback == 'sample':
                RL_loss += (- 0.01 * entropys[t] * mask_).sum()
            self.logs['critic_loss'].append((((r_ - v_) ** 2) * mask_).sum().item())

            total = total + np.sum(masks[t])
        self.logs['total'].append(total)

        self.RL_loss += RL_loss
        self.loss += RL_loss

        if 'test' not in self.env.splits:
            self.losses.append(self.loss.item() / self.episode_len)
            self.IL_losses.append(self.IL_loss.item() / self.episode_len)
            self.WeTA_losses.append(self.WeTA_loss.item() / self.episode_len)
            self.WaTA_losses.append(self.WaTA_loss.item() / self.episode_len)
            self.RL_losses.append(self.RL_loss.item() / self.episode_len)

        torch.cuda.empty_cache()

        return traj

    def test(self, use_dropout=False, feedback='argmax', allow_cheat=False):
        ''' Evaluate once on each instruction in the current environment '''
        if not allow_cheat: # permitted for purpose of calculating validation loss only
            assert feedback in ['argmax', 'sample'] # no cheating by using teacher at test time!
        self.feedback = feedback
        if use_dropout:
            self.encoder.train()
            self.decoder.train()
            self.critic.train()
            self.WeTA.train()
        else:
            self.encoder.eval()
            self.decoder.eval()
            self.critic.eval()
            self.WeTA.eval()
        with torch.no_grad():
            super(SCoA_Agent, self).test()

    def train(self, encoder_optimizer, decoder_optimizer, critic_optimizer, WeTA_optimizer,
              n_iters, feedback='teacher'):
        ''' Train for a given number of iterations '''
        assert feedback in self.feedback_options
        self.feedback = feedback
        self.encoder.train()
        self.decoder.train()
        self.critic.train()
        self.WeTA.train()

        self.losses = []
        self.WeTA_losses = []
        self.RL_losses = []
        self.WaTA_losses = []
        self.IL_losses = []
        for iter in range(1, n_iters + 1):
            encoder_optimizer.zero_grad()
            decoder_optimizer.zero_grad()
            critic_optimizer.zero_grad()
            WeTA_optimizer.zero_grad()

            self.rollout(iter)
            self.loss.backward()

            encoder_optimizer.step()
            decoder_optimizer.step()
            critic_optimizer.step()
            WeTA_optimizer.step()

    def save(self, encoder_path, decoder_path, critic_path, WeTA_path):
        ''' Snapshot models '''
        torch.save(self.encoder.state_dict(), encoder_path)
        torch.save(self.decoder.state_dict(), decoder_path)
        torch.save(self.critic.state_dict(), critic_path)
        torch.save(self.WeTA.state_dict(), WeTA_path)

    def load(self, encoder_path, decoder_path, critic_path, WeTA_path):
        ''' Loads parameters (but not training state) '''
        print("loading encoder, decoder, critic, WeTA")
        self.encoder.load_state_dict(torch.load(encoder_path))
        self.decoder.load_state_dict(torch.load(decoder_path))
        self.critic.load_state_dict(torch.load(critic_path))
        self.WeTA.load_state_dict(torch.load(WeTA_path))


import matplotlib.pyplot as plt

plt.ion()

def plot(scores, mean):
  plt.clf()
  plt.title('Statistics')
  plt.xlabel('Games')
  plt.ylabel('Score')
  plt.plot(scores)
  plt.plot(mean)
  plt.ylim(ymin=0)
  plt.text(len(scores)-1, scores[-1], str(scores[-1]))
  plt.text(len(mean)-1, mean[-1], str(mean[-1]))
  plt.pause(0.0001)
  plt.show()


"""
Greedy Word Swap with Word Importance Ranking
===================================================

When WIR method is set to ``unk``, this is a reimplementation of the search
method from the paper: Is BERT Really Robust?

A Strong Baseline for Natural Language Attack on Text Classification and
Entailment by Jin et. al, 2019. See https://arxiv.org/abs/1907.11932 and
https://github.com/jind11/TextFooler.
"""

import numpy as np
import torch
from torch.nn.functional import softmax

from textattack.goal_function_results import GoalFunctionResultStatus
from textattack.search_methods import SearchMethod
from textattack.shared.validators import (
    transformation_consists_of_word_swaps_and_deletions,
)

import math
import random
from textattack.metrics import Metric
#from textattack.constraints.semantics.sentence_encoders import UniversalSentenceEncoder
#use = UniversalSentenceEncoder()

class GreedyWordSwapWIR(SearchMethod):
    """An attack that greedily chooses from a list of possible perturbations in
    order of index, after ranking indices by importance.

    Args:
        wir_method: method for ranking most important words
        model_wrapper: model wrapper used for gradient-based ranking
    """

    def __init__(self, wir_method="unk"):
        self.wir_method = wir_method

    def _get_index_order(self, initial_text):
        """Returns word indices of ``initial_text`` in descending order of
        importance."""
        len_text = len(initial_text.words)

        if self.wir_method == "unk":
            leave_one_texts = [
                initial_text.replace_word_at_index(i, "[UNK]") for i in range(len_text)
            ]
            leave_one_results, search_over = self.get_goal_results(leave_one_texts)
            index_scores = np.array([result.score for result in leave_one_results])

        elif self.wir_method == "weighted-saliency":
            # first, compute word saliency
            leave_one_texts = [
                initial_text.replace_word_at_index(i, "[UNK]") for i in range(len_text)
            ]
            leave_one_results, search_over = self.get_goal_results(leave_one_texts)
            saliency_scores = np.array([result.score for result in leave_one_results])

            softmax_saliency_scores = softmax(
                torch.Tensor(saliency_scores), dim=0
            ).numpy()

            # compute the largest change in score we can find by swapping each word
            delta_ps = []
            for idx in range(len_text):
                transformed_text_candidates = self.get_transformations(
                    initial_text,
                    original_text=initial_text,
                    indices_to_modify=[idx],
                )
                if not transformed_text_candidates:
                    # no valid synonym substitutions for this word
                    delta_ps.append(0.0)
                    continue
                swap_results, _ = self.get_goal_results(transformed_text_candidates)
                score_change = [result.score for result in swap_results]
                if not score_change:
                    delta_ps.append(0.0)
                    continue
                max_score_change = np.max(score_change)
                delta_ps.append(max_score_change)

            index_scores = softmax_saliency_scores * np.array(delta_ps)

        elif self.wir_method == "delete":
            leave_one_texts = [
                initial_text.delete_word_at_index(i) for i in range(len_text)
            ]
            #print(leave_one_texts)
            leave_one_results, search_over = self.get_goal_results(leave_one_texts)
            index_scores = np.array([result.score for result in leave_one_results])
            #print(index_scores)

        elif self.wir_method == "gradient":
            victim_model = self.get_victim_model()
            index_scores = np.zeros(initial_text.num_words)
            grad_output = victim_model.get_grad(initial_text.tokenizer_input)
            gradient = grad_output["gradient"]
            word2token_mapping = initial_text.align_with_model_tokens(victim_model)
            for i, word in enumerate(initial_text.words):
                matched_tokens = word2token_mapping[i]
                #print(matched_tokens)
                if not matched_tokens:
                    index_scores[i] = 0.0
                else:
                    if matched_tokens!=None:
                    	matched_tokens=None
                    else:
                    	agg_grad = np.mean(gradient[matched_tokens], axis=0)
                    	index_scores[i] = np.linalg.norm(agg_grad, ord=1)

            search_over = False

        elif self.wir_method == "random":
            index_order = np.arange(len_text)
            np.random.shuffle(index_order)
            search_over = False
        else:
            raise ValueError(f"Unsupported WIR method {self.wir_method}")

        if self.wir_method != "random":
            index_order = (index_scores).argsort()[::-1]
            #index_order = (index_scores).argsort()
        return index_order, search_over
        #indices_to_modify=[]
    """
    def perform_search(self, initial_result):
        attacked_text = initial_result.attacked_text
        #print(attacked_text)
        # Sort words by order of importance
        index_order, search_over = self._get_index_order(attacked_text)
        print(index_order)
        i = 0
        cur_result = initial_result
        results = None
        while i < len(index_order) and not search_over:
            transformed_text_candidates = self.get_transformations(
                cur_result.attacked_text,
                original_text=initial_result.attacked_text,
                indices_to_modify=[index_order[i:i+4]],
            )
            #print(transformed_text_candidates)
            i += 1
            if len(transformed_text_candidates) == 0:
                continue
            results, search_over = self.get_goal_results(transformed_text_candidates)
            results = sorted(results, key=lambda x: -x.score)
            print("results[0].score")
            print(results[0].score)
            #print(type(results))  <class list>
            # Skip swaps which don't improve the score
            if results[0].score > cur_result.score:
                cur_result = results[0]
                #index_order, search_over = self._get_index_order(cur_result.attacked_text)
                #print("cur_result.score")
                #print(cur_result.score)
            else:
                continue
            # If we succeeded, return the index with best similarity.
            if cur_result.goal_status == GoalFunctionResultStatus.SUCCEEDED:
                best_result = cur_result
                # @TODO: Use vectorwise operations
                max_similarity = -float("inf")
                for result in results:
                    if result.goal_status != GoalFunctionResultStatus.SUCCEEDED:
                        break
                    candidate = result.attacked_text
                    try:
                        similarity_score = candidate.attack_attrs["similarity_score"]
                    except KeyError:
                        # If the attack was run without any similarity metrics,
                        # candidates won't have a similarity score. In this
                        # case, break and return the candidate that changed
                        # the original score the most.
                        break
                    if similarity_score > max_similarity:
                        max_similarity = similarity_score
                        best_result = result
                        print("best_result.score")
                return best_result

        return cur_result
    """
    def perform_search(self, initial_result):
        attacked_text = initial_result.attacked_text

        # Sort words by order of importance
        index_order, search_over = self._get_index_order(attacked_text)
        #print(index_order)
        # Starts Simulated Annealing
        inputlen=len(index_order)
        T =1000 # initial temperature
        Tmin = (math.pow(0.9,inputlen+3))*T  # lowest temperature
        #k =round(inputlen*0.6)  # internal iterations
        #k=12

        if inputlen < 21:
            k=14
        else:
            k=15

        #print(k)
        x = initial_result  # initialize x as original input text
        xin = initial_result  # initialize xin as original input text in cycle
        t = 0  # time,external iterations

        while T >= Tmin:
            index_order, search_over = self._get_index_order(x.attacked_text)
            weights=[]
            iw=0

            if t==0:
                weights=torch.Tensor([0.7,0.3])#第一次删除在最不重要与次不重要里面选
                weightsam=torch.multinomial(weights,1)
                listk=weightsam.numpy()

            elif t==inputlen-2:
                listk=[0]

            else:
                iw=0
                while iw  < len(index_order):

                    if 1-(1/len(index_order))*iw<0 or iw>min(2*t,len(index_order)):
                        w=0
                    else:
                        w=1-(1/len(index_order))*iw

                    #iw=iw*0.95
                    weights.append(w)
                    iw+=1

                weights=torch.Tensor(weights)
                #weightsam=torch.multinomial(weights,min(k,len(weights)))
                weightsam=torch.multinomial(weights,k,replacement=True)
                listk=weightsam.numpy()
            #listk=random.sample(range(0,min(t+1,k,len(index_order))),min(k,len(index_order),t+1))#第一次删除取0或1不重要的
            #print(weights)
            #listk=random.sample(range(0,3*t),min(k,3*t+1)
            #print(index_order)
            for indice in listk:
                y=xin.score
                if indice >= len(index_order):
                    #print("continue due to indice out of range")
                    continue
                moves = self.get_transformations(
                            x.attacked_text,
                            original_text=initial_result.attacked_text,
                            indices_to_modify=[index_order[indice]]
                            )
                # Skip words without candidates
                if len(moves) == 0:
                    continue
                xNew, _ = self.get_goal_results(moves)
                xNew_sorted = sorted(xNew, key=lambda x: -x.score)
                #yNew = self._aim_function(initial_result, xNew_sorted[0])
                yNew=xNew_sorted[0].score
                #print(xNew_sorted[0].score)
                #print("差为：")
                #print(yNew-y)
                d=yNew-y
                #print(d)
                if xNew_sorted[0].score!=0:
                    if yNew-y > 0:
                        xin = xNew_sorted[0]
                    else:
                    # metropolis principle
                        #print(d)
                        """
                        if (-d)>0.01 and (-d)<=0.1:
                            d=d/100
                        elif (-d)>0.001 and (-d)<=0.01:
                            d=d/10

                        if t<=0.3*inputlen:
                            p = math.exp(-((d)*(-1000000))/T)
                        elif t>0.3*inputlen and t<0.8*inputlen:
                            p = math.exp(-((d)*(-900000))/T)
                        else:
                            p = math.exp(-((d)*(-1100000))/T)

                        if t<=5:
                            p = math.exp(-((d)*(-900000))/T)
                        elif t>5 and t<12:
                            p = math.exp(-((d)*(-1000000))/T)
                        else:
                            p = math.exp(-((d)*(-450000))/T)

                        if len(initial_result.attacked_text.words)>16:
                            p=math.exp(-(60/T))
                        elif len(initial_result.attacked_text.words)<17 and len(initial_result.attacked_text.words)>9:
                            p = math.exp(-(100/T))
                        else:
                            p = math.exp(-(150/T))

                        if inputlen > 20:
                            p = math.exp(-((d)*(-1000000))/T)
                        if inputlen <=20 and inputlen >=10:
                            p = math.exp(-((d)*(-990000))/T)
                        else:
                            p = math.exp(-((d)*(-950000))/T)
                        """
                        p = math.exp(-((-d)*1000000)/T)
                        #print("T为：")
                        #print(T)
                        #print("p为：")
                        #print(p)
                        r = np.random.uniform(low=0, high=1)
                        #r=torch.rand(1).item()
                        #print(r)
                        if r < p:
                            xin = xNew_sorted[0]
                            #print("跳出")
                else:
                    continue
            t += 1
            T = 0.9*T
            #分段体现在降温函数上也可，跳出概率函数中的T改为T1即可
            #T = 3/(5*t)
            x=xin
            #semantics_score=use._sim_score(x.attacked_text)

            #print(x.attacked_text.tests)

        return x
        #f返回x1、x2，比较取得分最高
    """
    def perform_search(self, initial_result):
        beam_width=5
        index_order, search_over = self._get_index_order(initial_result.attacked_text)
        beam = [initial_result.attacked_text]
        best_result = initial_result
        while not best_result.goal_status == GoalFunctionResultStatus.SUCCEEDED:
            potential_next_beam = []
            for text in beam:
                index_order, search_over = self._get_index_order(text)
                transformations = self.get_transformations(
                    text,
                    original_text=initial_result.attacked_text,
                    indices_to_modify=[index_order[:beam_width]]
                )
                potential_next_beam += transformations

            if len(potential_next_beam) == 0:
                # If we did not find any possible perturbations, give up.
                return best_result
            results, search_over = self.get_goal_results(potential_next_beam)
            scores = np.array([r.score for r in results])
            if results[scores.argmax()].score!=0:
                best_result = results[scores.argmax()]
            if search_over:
                return best_result

            # Refill the beam. This works by sorting the scores
            # in descending order and filling the beam from there.
            best_indices = (-scores).argsort()[: beam_width]
            beam = [potential_next_beam[i] for i in best_indices]
            #print(beam)
        return best_result
    """
    def check_transformation_compatibility(self, transformation):
        """Since it ranks words by their importance, GreedyWordSwapWIR is
        limited to word swap and deletion transformations."""
        return transformation_consists_of_word_swaps_and_deletions(transformation)

    @property
    def is_black_box(self):
        if self.wir_method == "gradient":
            return False
        else:
            return True

    def extra_repr_keys(self):
        return ["wir_method"]


from django.core.management.base import BaseCommand
from django.core.files import File
from django.conf import settings
from muse.models import Album, Song
from os import walk, path
import muse.utl as utl

default_album_name = "loose songs"

class Command(BaseCommand):
    args = '<music_root_dir>'
    help = 'our help string comes here'

    def add_arguments(self, parser):
        parser.add_argument('music_root', nargs=1, type=str)

    def _populate_album_table(self, music_root):
        default_album = Album(name=default_album_name)
        default_album.save()
        for root, dirs, files in walk(music_root):
            for d in dirs:
                # Don't double add if rerunning command
                if not Album.objects.filter(name=d):
                    print(d)
                    new = Album(name=d)
                    new.save()
            # Only immediate subdirs of music_root are albums, so break
            break

    def _populate_song_table(self, music_root):
        for root, dirs, files in walk(music_root):
            # Only look for songs in music_root and immediate subdirs
            if root != music_root and path.dirname(root) != music_root:
                continue
            print(root)
            for fname in files:
                if not utl.is_music(fname):
                    continue
                if (root == music_root):
                    album_name = default_album_name
                else:
                    album_name = path.basename(root)
                album = Album.objects.get(name=album_name)
                full_path = path.join(root, fname)
                if not Song.objects.filter(name=fname):
                    print(album.name, full_path)
                    with open(full_path, 'rb') as f:
                        new = Song(name=fname, album=album, music_file=File(f))
                        new.save()

    def handle(self, *args, **options):
        self._populate_album_table(options['music_root'][0])
        print('----')
        self._populate_song_table(options['music_root'][0])


import time
import os
from reportlab.lib.enums import TA_JUSTIFY
from reportlab.lib.pagesizes import letter
from reportlab.platypus import SimpleDocTemplate, Paragraph, Spacer, Image
from reportlab.lib.styles import getSampleStyleSheet, ParagraphStyle
from reportlab.lib.units import inch
def pdf(data):
    route = os.getcwd() + "\\pacient.pdf"
    doc = SimpleDocTemplate(route, pagesize=letter,
                            rightMargin=72, leftMargin=72,
                            topMargin=72, bottomMargin=18)
    Story = []
    logotipo = os.getcwd() + "\\logo.png"
    pacient_name = data['usr_name']
    formatoFecha = time.ctime()
    imagen = Image(logotipo, 2 * inch, 1 * inch)
    Story.append(imagen)
    estilos = getSampleStyleSheet()
    estilos.add(ParagraphStyle(name='Justify', alignment=TA_JUSTIFY))
    texto = '%s' % formatoFecha
    Story.append(Paragraph(texto, estilos["Normal"]))
    Story.append(Spacer(1, 12))
    texto = 'Medical history of %s:' % pacient_name
    Story.append(Paragraph(texto, estilos["Normal"]))
    Story.append(Spacer(1, 12))
    dict_list = data['data']
    for dict_p in dict_list:
        texto = 'Hospital: %s' % dict_p['hospital']
        Story.append(Paragraph(texto, estilos["Normal"]))
        Story.append(Spacer(1, 12))
        texto = 'Doctor: %s' % dict_p['doctor']
        Story.append(Paragraph(texto, estilos["Normal"]))
        Story.append(Spacer(1, 12))
        texto = 'Especiality: %s' % dict_p['especiality']
        Story.append(Paragraph(texto, estilos["Normal"]))
        Story.append(Spacer(1, 12))
        texto = 'Observations: %s' % dict_p['details']
        Story.append(Paragraph(texto, estilos["Justify"]))
        Story.append(Spacer(1, 12))
        Story.append(Spacer(1, 12))
        Story.append(Spacer(1, 12))
    texto = 'Heippi developer team.'
    Story.append(Paragraph(texto, estilos["Normal"]))
    doc.build(Story)
    return route


#!/usr/bin/env python
##############################################################################
#
# diffpy.structure  by DANSE Diffraction group
#                   Simon J. L. Billinge
#                   (c) 2007 trustees of the Michigan State University.
#                   All rights reserved.
#
# File coded by:    Pavol Juhas
#
# See AUTHORS.txt for a list of people who contributed.
# See LICENSE_DANSE.txt for license information.
#
##############################################################################

"""Parser for extended CFG format used by atomeye"""

import sys
import re
import numpy
import six

from diffpy.structure import Structure
from diffpy.structure import StructureFormatError
from diffpy.structure.utils import isfloat
from diffpy.structure.parsers import StructureParser

# Constants ------------------------------------------------------------------

# Atomic Mass of elements
# This can be later when PeriodicTable package becomes available.

AtomicMass = {
    "H"  : 1.007947,    # 1 H hydrogen 1.007947
    "He" : 4.0026022,   # 2 He helium 4.0026022
    "Li" : 6.9412,      # 3 Li lithium 6.9412
    "Be" : 9.0121823,   # 4 Be beryllium 9.0121823
    "B"  : 10.8117,     # 5 B boron 10.8117
    "C"  : 12.01078,    # 6 C carbon 12.01078
    "N"  : 14.00672,    # 7 N nitrogen 14.00672
    "O"  : 15.99943,    # 8 O oxygen 15.99943
    "F"  : 18.99840325, # 9 F fluorine 18.99840325
    "Ne" : 20.17976,    # 10 Ne neon 20.17976
    "Na" : 22.9897702,  # 11 Na sodium 22.9897702
    "Mg" : 24.30506,    # 12 Mg magnesium 24.30506
    "Al" : 26.9815382,  # 13 Al aluminium 26.9815382
    "Si" : 28.08553,    # 14 Si silicon 28.08553
    "P"  : 30.9737612,  # 15 P phosphorus 30.9737612
    "S"  : 32.0655,     # 16 S sulfur 32.0655
    "Cl" : 35.4532,     # 17 Cl chlorine 35.4532
    "Ar" : 39.9481,     # 18 Ar argon 39.9481
    "K"  : 39.09831,    # 19 K potassium 39.09831
    "Ca" : 40.0784,     # 20 Ca calcium 40.0784
    "Sc" : 44.9559108,  # 21 Sc scandium 44.9559108
    "Ti" : 47.8671,     # 22 Ti titanium 47.8671
    "V"  : 50.94151,    # 23 V vanadium 50.94151
    "Cr" : 51.99616,    # 24 Cr chromium 51.99616
    "Mn" : 54.9380499,  # 25 Mn manganese 54.9380499
    "Fe" : 55.8452,     # 26 Fe iron 55.8452
    "Co" : 58.9332009,  # 27 Co cobalt 58.9332009
    "Ni" : 58.69342,    # 28 Ni nickel 58.69342
    "Cu" : 63.5463,     # 29 Cu copper 63.5463
    "Zn" : 65.4094,     # 30 Zn zinc 65.4094
    "Ga" : 69.7231,     # 31 Ga gallium 69.7231
    "Ge" : 72.641,      # 32 Ge germanium 72.641
    "As" : 74.921602,   # 33 As arsenic 74.921602
    "Se" : 78.963,      # 34 Se selenium 78.963
    "Br" : 79.9041,     # 35 Br bromine 79.9041
    "Kr" : 83.7982,     # 36 Kr krypton 83.7982
    "Rb" : 85.46783,    # 37 Rb rubidium 85.46783
    "Sr" : 87.621,      # 38 Sr strontium 87.621
    "Y"  : 88.905852,   # 39 Y yttrium 88.905852
    "Zr" : 91.2242,     # 40 Zr zirconium 91.2242
    "Nb" : 92.906382,   # 41 Nb niobium 92.906382
    "Mo" : 95.942,      # 42 Mo molybdenum 95.942
    "Tc" : 98.0,        # 43 Tc technetium 98
    "Ru" : 101.072,     # 44 Ru ruthenium 101.072
    "Rh" : 102.905502,  # 45 Rh rhodium 102.905502
    "Pd" : 106.421,     # 46 Pd palladium 106.421
    "Ag" : 107.86822,   # 47 Ag silver 107.86822
    "Cd" : 112.4118,    # 48 Cd cadmium 112.4118
    "In" : 114.8183,    # 49 In indium 114.8183
    "Sn" : 118.7107,    # 50 Sn tin 118.7107
    "Sb" : 121.7601,    # 51 Sb antimony 121.7601
    "Te" : 127.603,     # 52 Te tellurium 127.603
    "I"  : 126.904473,  # 53 I iodine 126.904473
    "Xe" : 131.2936,    # 54 Xe xenon 131.2936
    "Cs" : 132.905452,  # 55 Cs caesium 132.905452
    "Ba" : 137.3277,    # 56 Ba barium 137.3277
    "La" : 138.90552,   # 57 La lanthanum 138.90552
    "Ce" : 140.1161,    # 58 Ce cerium 140.1161
    "Pr" : 140.907652,  # 59 Pr praseodymium 140.907652
    "Nd" : 144.243,     # 60 Nd neodymium 144.243
    "Pm" : 145.0,       # 61 Pm promethium 145
    "Sm" : 150.363,     # 62 Sm samarium 150.363
    "Eu" : 151.9641,    # 63 Eu europium 151.9641
    "Gd" : 157.253,     # 64 Gd gadolinium 157.253
    "Tb" : 158.925342,  # 65 Tb terbium 158.925342
    "Dy" : 162.5001,    # 66 Dy dysprosium 162.5001
    "Ho" : 164.930322,  # 67 Ho holmium 164.930322
    "Er" : 167.2593,    # 68 Er erbium 167.2593
    "Tm" : 168.934212,  # 69 Tm thulium 168.934212
    "Yb" : 173.043,     # 70 Yb ytterbium 173.043
    "Lu" : 174.9671,    # 71 Lu lutetium 174.9671
    "Hf" : 178.492,     # 72 Hf hafnium 178.492
    "Ta" : 180.94791,   # 73 Ta tantalum 180.94791
    "W"  : 183.841,     # 74 W tungsten 183.841
    "Re" : 186.2071,    # 75 Re rhenium 186.2071
    "Os" : 190.233,     # 76 Os osmium 190.233
    "Ir" : 192.2173,    # 77 Ir iridium 192.2173
    "Pt" : 195.0782,    # 78 Pt platinum 195.0782
    "Au" : 196.966552,  # 79 Au gold 196.966552
    "Hg" : 200.592,     # 80 Hg mercury 200.592
    "Tl" : 204.38332,   # 81 Tl thallium 204.38332
    "Pb" : 207.21,      # 82 Pb lead 207.21
    "Bi" : 208.980382,  # 83 Bi bismuth 208.980382
    "Po" : 209.0,       # 84 Po polonium 209
    "At" : 210.0,       # 85 At astatine 210
    "Rn" : 222.0,       # 86 Rn radon 222
    "Fr" : 223.0,       # 87 Fr francium 223
    "Ra" : 226.0,       # 88 Ra radium 226
    "Ac" : 227.0,       # 89 Ac actinium 227
    "Th" : 232.03811,   # 90 Th thorium 232.03811
    "Pa" : 231.035882,  # 91 Pa protactinium 231.035882
    "U"  : 238.028913,  # 92 U uranium 238.028913
    "Np" : 237.0,       # 93 Np neptunium 237
    "Pu" : 244.0,       # 94 Pu plutonium 244
    "Am" : 243.0,       # 95 Am americium 243
    "Cm" : 247.0,       # 96 Cm curium 247
    "Bk" : 247.0,       # 97 Bk berkelium 247
    "Cf" : 251.0,       # 98 Cf californium 251
    "Es" : 252.0,       # 99 Es einsteinium 252
    "Fm" : 257.0,       # 100 Fm fermium 257
    "Md" : 258.0,       # 101 Md mendelevium 258
    "No" : 259.0,       # 102 No nobelium 259
    "Lr" : 262.0,       # 103 Lr lawrencium 262
    "Rf" : 261.0,       # 104 Rf rutherfordium 261
    "Db" : 262.0,       # 105 Db dubnium 262
    "Sg" : 266.0,       # 106 Sg seaborgium 266
    "Bh" : 264.0,       # 107 Bh bohrium 264
    "Hs" : 277.0,       # 108 Hs hassium 277
    "Mt" : 268.0,       # 109 Mt meitnerium 268
    "Ds" : 281.0,       # 110 Ds darmstadtium 281
    "Rg" : 272.0,       # 111 Rg roentgenium 272
}

# ----------------------------------------------------------------------------

class P_xcfg(StructureParser):
    """Parser for AtomEye extended CFG format.

    cluster_boundary -- width of boundary around corners of non-periodic
                        cluster to avoid PBC effects in atomeye
    """

    cluster_boundary = 2

    def __init__(self):
        StructureParser.__init__(self)
        self.format = "xcfg"
        return

    def parseLines(self, lines):
        """Parse list of lines in PDB format.

        Return Structure object or raise StructureFormatError.
        """
        xcfg_Number_of_particles = None
        xcfg_A = None
        xcfg_H0 = numpy.zeros((3,3), dtype=float)
        xcfg_H0_set = numpy.zeros((3,3), dtype=bool)
        xcfg_NO_VELOCITY = False
        xcfg_entry_count = None
        p_nl = 0
        p_auxiliary_re = re.compile(r"^auxiliary\[(\d+)\] =")
        p_auxiliary = {}
        stru = Structure()
        # ignore trailing blank lines
        stop = len(lines)
        for line in reversed(lines):
            if line.strip():
                break
            stop -= 1
        # iterator over the valid data lines
        ilines = iter(lines[:stop])
        try:
            # read XCFG header
            for line in ilines:
                p_nl += 1
                stripped_line = line.strip()
                # blank lines and lines starting with # are ignored
                if stripped_line == "" or line[0] == '#':
                    continue
                elif xcfg_Number_of_particles is None:
                    if line.find("Number of particles =") != 0:
                        emsg = ("%d: first line must " +
                                "contain 'Number of particles ='") % p_nl
                        raise StructureFormatError(emsg)
                    xcfg_Number_of_particles = int(line[21:].split(None, 1)[0])
                    p_natoms = xcfg_Number_of_particles
                elif line.find("A =") == 0:
                    xcfg_A = float(line[3:].split(None, 1)[0])
                elif line.find("H0(") == 0:
                    i, j = (int(line[3]) - 1, int(line[5]) - 1)
                    xcfg_H0[i,j] = float(line[10:].split(None, 1)[0])
                    xcfg_H0_set[i,j] = True
                elif line.find(".NO_VELOCITY.") == 0:
                    xcfg_NO_VELOCITY = True
                elif line.find("entry_count =") == 0:
                    xcfg_entry_count = int(line[13:].split(None, 1)[0])
                elif p_auxiliary_re.match(line):
                    m = p_auxiliary_re.match(line)
                    idx = int(m.group(1))
                    p_auxiliary[idx] = line[m.end():].split(None, 1)[0]
                else:
                    break
            # check header for consistency
            if numpy.any(xcfg_H0_set == False):
                emsg = "H0 tensor is not properly defined"
                raise StructureFormatError(emsg)
            p_auxnum = len(p_auxiliary) and max(p_auxiliary.keys())+1
            for i in range(p_auxnum):
                if not i in p_auxiliary:
                    p_auxiliary[i] = "aux%d" % i
            sorted_aux_keys = sorted(p_auxiliary.keys())
            if p_auxnum != 0:
                stru.xcfg = {
                    'auxiliaries' : [ p_auxiliary[k]
                                      for k in sorted_aux_keys ]
                }
            ecnt = len(p_auxiliary) + (3 if xcfg_NO_VELOCITY else 6)
            if ecnt != xcfg_entry_count:
                emsg = ("%d: auxiliary fields are "
                        "not consistent with entry_count") % p_nl
                raise StructureFormatError(emsg)
            # define proper lattice
            stru.lattice.setLatBase(xcfg_H0)
            # here we are inside the data block
            p_element = None
            for line in ilines:
                p_nl += 1
                words = line.split()
                # ignore atom mass
                if len(words) == 1 and isfloat(words[0]):
                    continue
                # parse element allowing empty symbol
                elif len(words) <= 1:
                    w = line.strip()
                    p_element = w[:1].upper() + w[1:].lower()
                elif len(words) == xcfg_entry_count and p_element is not None:
                    fields = [float(w) for w in words]
                    xyz = [xcfg_A * xi for xi in fields[:3]]
                    stru.addNewAtom(p_element, xyz=xyz)
                    a = stru[-1]
                    _assign_auxiliaries(a, fields, auxiliaries=p_auxiliary,
                                        no_velocity=xcfg_NO_VELOCITY)
                else:
                    emsg = "%d: invalid record" % p_nl
                    raise StructureFormatError(emsg)
            if len(stru) != p_natoms:
                emsg = "expected %d atoms, read %d" % (p_natoms, len(stru))
                raise StructureFormatError(emsg)
        except (ValueError, IndexError):
            emsg = "%d: file is not in XCFG format" % p_nl
            exc_type, exc_value, exc_traceback = sys.exc_info()
            e = StructureFormatError(emsg)
            six.reraise(StructureFormatError, e, exc_traceback)
        return stru

    def toLines(self, stru):
        """Convert Structure stru to a list of lines in XCFG atomeye format.

        Return list of strings.
        """
        if len(stru) == 0:
            emsg = "cannot convert empty structure to XCFG format"
            raise StructureFormatError(emsg)
        lines = []
        lines.append("Number of particles = %i" % len(stru))
        # figure out length unit A
        allxyz = numpy.array([a.xyz for a in stru])
        lo_xyz = allxyz.min(axis=0)
        hi_xyz = allxyz.max(axis=0)
        max_range_xyz = (hi_xyz-lo_xyz).max()
        if numpy.allclose(stru.lattice.abcABG(), (1, 1, 1, 90, 90, 90)):
            max_range_xyz += self.cluster_boundary
        # range of CFG coordinates must be less than 1
        p_A = numpy.ceil(max_range_xyz + 1.0e-13)
        # atomeye draws rubbish when boxsize is less than 3.5
        hi_ucvect = max([numpy.sqrt(numpy.dot(v,v)) for v in stru.lattice.base])
        if hi_ucvect*p_A < 3.5:
            p_A = numpy.ceil(3.5 / hi_ucvect)
        lines.append("A = %.8g Angstrom" % p_A)
        # how much do we need to shift the coordinates?
        p_dxyz = numpy.zeros(3, dtype=float)
        for i in range(3):
            if lo_xyz[i]/p_A < 0.0 or hi_xyz[i]/p_A >= 1.0 \
            or (lo_xyz[i] == hi_xyz[i] and lo_xyz[i] == 0.0) :
                p_dxyz[i] = 0.5 - (hi_xyz[i]+lo_xyz[i])/2.0/p_A
        # H0 tensor
        for i in range(3):
            for j in range(3):
                lines.append("H0(%i,%i) = %.8g A" %
                             (i + 1, j + 1, stru.lattice.base[i, j]))
        # get out for empty structure
        if len(stru) == 0: return lines
        a_first = stru[0]
        p_NO_VELOCITY = "v" not in a_first.__dict__
        if p_NO_VELOCITY:
            lines.append(".NO_VELOCITY.")
        # build a p_auxiliaries list of (aux_name,atom_expression) tuples
        # if stru came from xcfg file, it would store original auxiliaries in
        # xcfg dictionary
        try:
            p_auxiliaries = [ (aux, "a."+aux)
                for aux in stru.xcfg['auxiliaries'] ]
        except AttributeError:
            p_auxiliaries = []
        # add occupancy if any atom has nonunit occupancy
        for a in stru:
            if a.occupancy != 1.0:
                p_auxiliaries.append(('occupancy', 'a.occupancy'))
                break
        # add temperature factor with as many terms as needed
        # check whether all temperature factors are zero or isotropic
        p_allUzero = True
        p_allUiso = True
        for a in stru:
            if p_allUzero and numpy.any(a.U != 0.0):
                p_allUzero = False
            if not numpy.all(a.U == a.U[0,0]*numpy.identity(3)):
                p_allUiso = False
                # here p_allUzero must be false
                break
        if p_allUzero:
            pass
        elif p_allUiso:
            p_auxiliaries.append(('Uiso', 'uflat[0]'))
        else:
            p_auxiliaries.extend([('U11', 'uflat[0]'),
                                  ('U22', 'uflat[4]'),
                                  ('U33', 'uflat[8]')])
            # check if there are off-diagonal elements
            allU = numpy.array([a.U for a in stru])
            if numpy.any(allU[:,0,1] != 0.0):
                p_auxiliaries.append(('U12', 'uflat[1]'))
            if numpy.any(allU[:,0,2] != 0.0):
                p_auxiliaries.append(('U13', 'uflat[2]'))
            if numpy.any(allU[:,1,2] != 0.0):
                p_auxiliaries.append(('U23', 'uflat[5]'))
        # count entries
        p_entry_count = (3 if p_NO_VELOCITY else 6) + len(p_auxiliaries)
        lines.append("entry_count = %d" % p_entry_count)
        # add auxiliaries
        for i in range(len(p_auxiliaries)):
            lines.append("auxiliary[%d] = %s [au]" % (i, p_auxiliaries[i][0]))
        # now define entry format efmt for representing atom properties
        fmwords = ["{pos[0]:.8g}", "{pos[1]:.8g}", "{pos[2]:.8g}"]
        if not p_NO_VELOCITY:
            fmwords += ["{v[0]:.8g}", "{v[1]:.8g}", "{v[2]:.8g}"]
        fmwords += (('{' + e + ':.8g}') for p, e in p_auxiliaries)
        efmt = ' '.join(fmwords)
        # we are ready to output atoms:
        lines.append("")
        p_element = None
        for a in stru:
            if a.element != p_element:
                p_element = a.element
                lines.append("%.4f" % AtomicMass.get(p_element, 0.0))
                lines.append(p_element)
            pos = a.xyz / p_A + p_dxyz
            v = None if p_NO_VELOCITY else a.v
            uflat = numpy.ravel(a.U)
            entry = efmt.format(pos=pos, v=v, uflat=uflat, a=a)
            lines.append(entry)
        return lines

# End of class P_xcfg

# Routines -------------------------------------------------------------------

def getParser():
    return P_xcfg()

# Local Helpers --------------------------------------------------------------

def _assign_auxiliaries(a, fields, auxiliaries, no_velocity):
    """\
    Assing auxiliary properties for Atom object when reading CFG format.

    Parameters
    ----------
    a : Atom
        The Atom instance for which the auxiliary properties need to be set.
    fields : list
        Floating point values for the current row of the processed CFG file.
    auxiliaries : dict
        Dictionary of zero-based indices and names of auxiliary properties
        defined in the CFG format.
    no_velocity : bool
        When `False` set atom velocity `a.v` to `fields[3:6]`.
        Use `fields[3:6]` for auxiliary values otherwise.

    No return value.
    """
    if not no_velocity:
        a.v = numpy.asarray(fields[3:6], dtype=float)
    auxfirst = 3 if no_velocity else 6
    for i, prop in auxiliaries.items():
        value = fields[auxfirst + i]
        if prop == "Uiso":
            a.Uisoequiv = value
        elif prop == "Biso":
            a.Bisoequiv = value
        elif prop[0] in 'BU' and all(d in '123' for d in prop[1:]):
            nm = (prop if prop[1] <= prop[2]
                  else prop[0] + prop[2] + prop[1])
            a.anisotropy = True
            setattr(a, nm, value)
        else:
            setattr(a, prop, value)
    return


# coding:utf-8
import sys
import re
from xsqlmb.api.logstash.utils.dt_tool import get_pydt_based_logdt
from xsqlmb.api.logstash.cfgs.configs import WAF_ACCESS_LOG_SQL_TABLE, \
    WAF_ALERT_LOG_SQL_TABLE, WAF_ALERT_LOG_DETAILED_SQL_TABLE
#from utils.django_module import django_setup
try:
    from xsqlmb.src.cfgs.logConfig import logging
except:
    import logging

from uuid import uuid4
from xsqlmb.api.logstash.scripts.extract_log_f_mongo import ExtractLogFromMongo
from xsqlmb.src.dao.exutil import MutiTypesInsets2SqlClass
#from xsqlmb.src.ltool.sqlconn import from_sql_get_data, sql_action
from xsqlmb.api.logstash.utils.get_table_columns import get_waf_alert_log_columns, get_waf_access_log_columns
from ..cfgs.configs import WAF_ACCESS_LOG_SQL_TABLE, WAF_ALERT_LOG_SQL_TABLE

_waf_columns = get_waf_alert_log_columns()

class LogToSql():

    def __init__(self, MAX_INSERT_NUM=500):
        self.MAX_INSERT_NUM=MAX_INSERT_NUM

    def init_sql_logdb(self):
        pass

    def get_latest_accsslog(self):
        datas, line = ExtractLogFromMongo(table_name=WAF_ACCESS_LOG_SQL_TABLE).get_access_logs()
        return datas

    def get_latest_modseclog(self):
        return ExtractLogFromMongo(table_name=WAF_ALERT_LOG_SQL_TABLE).modseclog_to_detaild()

    def many_insert2_accesslog(self, nad_datas):
        from django.core.paginator import Paginator
        p = Paginator(nad_datas, self.MAX_INSERT_NUM) # 分页列别
        page_count = p.num_pages  # 总页数
        seccess_insert_num = 0

        # from xsqlmb.api.logstash.utils.get_table_columns import get_waf_access_log_columns
        cols = get_waf_access_log_columns()
        _columns = "`" + "`, `".join(cols) + "`"
        _keys = cols

        for x in [x+1 for x in range(page_count)]:
            nad_list = list(p.page(x).object_list)
            from xsqlmb.src.dao.exutil import MutiTypesInsets2SqlClass

            MutiTypesInsets2SqlClass(table_name = WAF_ACCESS_LOG_SQL_TABLE).arrays2sql2(
            nad_list, columns_order=_columns, keys_list=_keys)
            seccess_insert_num += len(nad_list)

        return seccess_insert_num

    def accesslog_to_sql(self, local=False):
        nad_datas = []
        _accesslog_datas = self.get_latest_accsslog()
        for x in _accesslog_datas:
            obj = x.copy()
            try:
                obj["time_local"] = get_pydt_based_logdt(re.match("(.*?)\s(.*)", obj["time_local"]).group(1))
                obj["timestamp"] = obj["time_local"]

                if "request_url" not in obj.keys():
                    obj["request_url"] = obj["url"]

                if "request_id" not in obj.keys():
                    obj["request_id"] = uuid4()

                obj["upstream_response_time"] = "0.01" if obj["upstream_response_time"] == "-" else "0.0"
                obj["request_time"] = "0.01" if obj["upstream_response_time"] == "-" else "0.0"
            except:
                logging.error("Error:存在AccessLog日志不一样的正则 " + obj["time_local"])

                continue
            nad_datas.append(obj)
        seccess_insert_num = self.many_insert2_accesslog(nad_datas)

        logging.info("插入【" + str(seccess_insert_num)  +"】条新数据到访问日志SQL数据库成功")

    def modseclog_to_sql(self):
        # from datetime import datetime
        # default_time = datetime(1995, 8, 14)
        nad_datas = self.get_latest_modseclog()

        from django.core.paginator import Paginator
        p = Paginator(nad_datas, self.MAX_INSERT_NUM)  # 分页列别
        page_count = p.num_pages  # 总页数
        seccess_insert_num = 0

        from xsqlmb.api.logstash.utils.get_table_columns import get_waf_alert_log_columns
        cols = get_waf_alert_log_columns()

        _columns = "`" + "`, `".join(cols) + "`"
        _keys = cols

        for x in [x + 1 for x in range(page_count)]:
            nad_list = list(p.page(x).object_list)
            try:
                _insert_num = MutiTypesInsets2SqlClass(table_name=WAF_ALERT_LOG_SQL_TABLE).arrays2sql2(
                    nad_list, columns_order=_columns, keys_list=_keys)
                seccess_insert_num += _insert_num

            finally:
                import json

                # _detailed_list = [ [x["audit_logid"], json.dumps(x)] for x in nad_list ]
                # MutiTypesInsets2SqlClass(table_name=WAF_ALERT_LOG_DETAILED_SQL_TABLE).arrays2sql(
                #     _detailed_list, columns_order="`audit_logid`,`detaild`"
                # )

                _detailed_list = [dict(audit_logid=x["audit_logid"], detaild=json.dumps(x) ) for x in nad_list]

                from xsqlmb.src.ltool.mongo import MongoConn
                from xsqlmb.api.logstash.cfgs.configs import WAF_ALERT_LOG_DETAILED_SQL_TABLE
                MongoConn().insert_data(WAF_ALERT_LOG_DETAILED_SQL_TABLE, _detailed_list)

        logging.info("插入【" + str(seccess_insert_num) + "】条新数据到告警日志SQL数据库成功")


import bpy
import struct

# ExportHelper is a helper class, defines filename and
# invoke() function which calls the file selector.
from bpy_extras.io_utils import ExportHelper
from bpy.props import StringProperty, BoolProperty, EnumProperty
from bpy.types import Operator

class Exportshp0(bpy.types.Operator, ExportHelper):
    """Export SHP0"""
    bl_idname = "export.shp0"  # important since its how bpy.ops.import_test.some_data is constructed
    bl_label = "Export shp0"
    filename_ext = ".shp0"

    def write_file_header(context, f, setting):
        f.write(struct.pack(">4s", b"SHP0")) # magic number
        f.write(struct.pack(">i", 0)) # placeholder for file length
        f.write(struct.pack(">i", setting.version)) # version number
        f.write(struct.pack(">i", 0)) # offset to brres file
        f.write(struct.pack(">4i", 0,0,0,0)) # placeholder for section offsets
        f.write(struct.pack(">i", 0)) #placeholder for offset to file name location in file

        #f.write(struct.pack(">4s", setting.filename))

    def write_shp0_header(context, f, setting):
        f.write(struct.pack(">i", 0))
        f.write(struct.pack(">H", setting.frame))
        f.write(struct.pack(">H", len(context.object.vertex_groups)))
        f.write(struct.pack(">i", setting.loop))
        #need to check if works bc loop is bool => correctly translated to 0x00/0x01 ?

    def update_file_length(context, f, setting, offset_placement):
        length = f.tell()
        f.seek(offset_placement, 0)
        f.write(struct.pack(">i", length))
        offset_placement += 4
        f.seek(length, 0)

    def write_anim_data(context, f, setting, index):

        if context.object.vertex_groups.active_index != -1:
            #strange way to get the value: enable='OPT_A' (str) so enable[4]= 'A'
            #ord() to get the ascii value then -64 to get 1 => 0x02
            enable_value = 2**(ord(context.scene.animGroup[index].enable[4]) - 64)
            f.write(struct.pack(">i", enable_value)) #correct 0x0_ value ?
        else:
            f.write(struct.pack(">i", 0))
            print("No vertex group found, exported as 0")
        index += 1

        f.write(struct.pack(">i", 0)) #placeholder for vg name (name of the target vertex set) offset
        f.write(struct.pack(">H", index))
        f.write(struct.pack(">H", len(context.object.data.shape_keys.key_blocks)))

        f.write(struct.pack(">i", 0)) #placeholder for 0x0C
        f.write(struct.pack(">i", 0)) #placeholder for 0x10
        f.write(struct.pack(">i", 0))
        for i in range(len(context.object.data.shape_keys.key_blocks)):
            f.write(struct.pack(">i", 0)) #placeholder for 0x14 offsets

    def write_anim_entry(self, context, f, setting):
        obj = context.active_object
        #'if sk is not None:' not needed bc already checked in write_shp0_file
        sk = obj.data.shape_keys
        anim = sk.animation_data
        if anim is not None:
            anim = anim.action
            if anim is not None:
                for fcu in anim.fcurves:
                    f.write(struct.pack(">H", 0)) #nb kf
                    f.write(struct.pack(">H", 0)) #padding?
                    f.write(struct.pack(">f", 0)) #float
                    for kf in fcu.keyframe_points:
                        f.write(struct.pack(">f", sk.key_blocks.frame)) #kf.index, surely not expected value
                        f.write(struct.pack(">f", sk.key_blocks.value)) #kf.value
                        f.write(struct.pack(">f", evaluate(kf)))
        else:
            self.report({'ERROR'}, "No animation found on active object")
            return {'CANCELLED'}

        return {'FINISHED'}

        #write_kf(context, f, setting)

    def write_vertex_data_names(context, f, setting):    #section 1
        Exportshp0.update_file_length(context, f, setting, 4)

    def write_shp0_file(self, context, setting):
        print("running write_shp0_file...")
        with open(self.filepath, 'wb') as f:
            nb = 4
            Exportshp0.write_file_header(context, f, setting)
            Exportshp0.write_shp0_header(context, f, setting)

            #Exportshp0.update_file_length(context, f, setting, nb*4+16)
            #Exportshp0.update_file_length(context, f, setting, nb*4+16)

            if context.object.data.shape_keys is not None:
                index = 0
                for sk in context.object.data.shape_keys.key_blocks:
                    Exportshp0.write_anim_data(context, f, setting, index)
                    Exportshp0.write_anim_entry(self, context, f, setting)
            else:
                self.report({'ERROR'}, "No shape key found on active object")
                return {'CANCELLED'}

            #for animations in animList
                #write anim header
                #write anim entry header
                #for kf in kfList
                    #write frame number, anim value, hermite slope interpolation

            Exportshp0.update_file_length(context, f, setting, 4)
        self.report({'INFO'}, ".shp0 exported successfully!")
        return {'FINISHED'}

    def execute(self, context):
        return Exportshp0.write_shp0_file(self, context, context.scene.addonProps)

class Anim(bpy.types.PropertyGroup):
    name : bpy.props.StringProperty(name="Name")
    lock : bpy.props.BoolProperty(name='Lock', default=False, description="So that you don't accidentally delete your animation (even if UNDO works)")
    enable: bpy.props.EnumProperty(name="Enable", description="Enable Update",
    items=(
        ('OPT_A', "Enable position update", "0x02"),
        ('OPT_B', "Enable normal update", "0x04"),
        ('OPT_C', "Enable vertex color update", "0x08"),
    ),
    default='OPT_A')

class settings(bpy.types.PropertyGroup):

    #File variables
    filename: bpy.props.StringProperty(name="File Name", description="name under which the file will be exported")
    loop : bpy.props.BoolProperty(name='Loop', default=False, description="Does the animation should loop?")
    version: bpy.props.IntProperty(name='Version', min = 1, max = 15, default=4,  description="Version number. It is recommended to leave it at 4, other versions are unknown")
    frame: bpy.props.IntProperty(name='Number of frames', default=58,  description="Number of frames of the animation")

class AnimManager(bpy.types.Operator):
    bl_idname = "anim.update"
    bl_label = "Update Animation"
    bl_options = {'REGISTER', 'UNDO'}

    def execute(self, context):
        scene = context.scene
        groups = context.object.vertex_groups

        if(len(groups) > scene.animIndex):
            for i in range (len(groups) - scene.animIndex):
                AnimManager.addAnim(self, context)
        elif(len(groups) < scene.animIndex):
            for i in range (scene.animIndex - len(groups)):
                AnimManager.removeAnim(self, context)
        return {'FINISHED'}

    def addAnim(self, context):
        scene = context.scene
        newAnim = scene.animGroup.add()
        newAnim.name = "Animation " + str(len(scene.animGroup) - 1)
        scene.animIndex = len(scene.animGroup) - 1
        return {'FINISHED'}

    def removeAnim(self, context):
        scene = context.scene
        animIndex = scene.animIndex
        if(animIndex >= 0 and not scene.animGroup[animIndex].lock):
            scene.animGroup.remove(animIndex)
            scene.animIndex = min(animIndex, len(scene.animGroup) - 1)
            return {'FINISHED'}
        else:
            return {'CANCELLED'}

class EditorPanel(bpy.types.Panel):
    """Creates a Panel in the Object properties window"""
    bl_label = "SHP0 Editor"
    bl_idname = "OBJECT_PT_editor"
    bl_space_type = 'VIEW_3D'
    bl_region_type = 'UI'
    bl_category = "SHP0 Editor"

    def draw(self, context):
        layout = self.layout
        obj = context.object
        scene = context.scene

        layout.label(text="Vertex Morph Editor", icon='RENDER_ANIMATION')
        layout.label(text="Active object is: " + obj.name)

        param = scene.addonProps
        anim = scene.animGroup

        groups = obj.vertex_groups

        layout.prop(param, "filename")
        row = layout.row()
        row.prop(param, "version")
        row.prop(param, "loop")
        col = layout.column()
        col.prop(param, "frame")

        col.label(text="Current Vertex Group : " + str(groups.active_index))

        layout.operator('export.shp0')
        layout.operator('anim.update')

        if(groups.active_index == -1):
            col.label(text="Please add a vertex group first")
        elif(groups.active_index > scene.animIndex):
            col.label(text="Please Update Animation first")
        else:
            col.prop(anim[groups.active_index], "enable")

    #how to use obj datas:
        #me = obj.data
        #groups = [obj.vertex_groups.active_index]
        #bpy.ops.object.mode_set(mode='OBJECT')
        #for v in me.vertices:
            #for g in v.groups:
                #if g.group in groups:

    #def execute(self, context):
        #return write_shp0_file(context, self.filepath, self.use_setting)

classes = [AnimManager, Exportshp0, Anim, settings, EditorPanel]

def register():
    for cl in classes:
        bpy.utils.register_class(cl)

    bpy.types.Scene.addonProps = bpy.props.PointerProperty(type=settings)
    bpy.types.Scene.animGroup = bpy.props.CollectionProperty(type=Anim)
    bpy.types.Scene.animIndex = bpy.props.IntProperty()

def unregister():
    for cl in classes:
        bpy.utils.unregister_class(cl)

    del bpy.types.Scene.addonProps

    del bpy.types.Scene.animGroup
    del bpy.types.Scene.animIndex

if __name__ == "__main__":
    register()

'''
roadmap:
    + is done
    * partially done
    - needs to be done

    Priorities:
        verify is animation data/entry correctly written to file
        section 1
        update offsets

    section 0:
        file header (just need length of file update with                                           +
        ftell() then fseek(0x04) once all the file is written)

        shp0 header                                                                                +

        Animation Data
            use of shape keys for animation entries
            0x00 bitflags                                                                           +
            (but still need the update animation button, hopefully find a way to get rid of it)
            and better coord between vg size and animIndex (off by 1 when size vg -1)               *

            0x04/8? name offset to section one vg name                                              *
            0x0A number of shape keys   (len(bpy.types.ShapeKey))                                   *

            0x0C-0x10-0x14 TBD                                                                      -

        Animation Entry
            "header"
                this shape key number of kf                                                         +
                padding                                                                             -
                1/(max kf number)                                                                   +
            followed by kf entries: (size N*12)
                frame number                                                                        +
                shape key animation value (0-1 morph)                                               +
                interpolation tangent value (f-curve or shapeKey.interpolation data struct?)        *

    section 1:
        list of vertex group names                                                                  -

    change animGroup to shape keys instead of vg ?

    Animation Data <=> shape keys
    Animations Entry <=> fcurve of shape key (animation data)
    keyframe <=> keyframe of fcurve (animation entry)

'''


#!/usr/bin/env python3
"""Find all intersections between chains and transcripts CDS.

Requires a bed and a chain file as input.
Produces the following table:
chain_id<tab>comma-separated list of overlapped genes.
"""
import sys
from collections import defaultdict
from version import __version__

try:  # for robustness
    from modules.common import flatten
except ImportError:
    from common import flatten

__author__ = "Bogdan Kirilenko, 2020."
__email__ = "bogdan.kirilenko@senckenberg.de"
__credits__ = ["Michael Hiller", "Virag Sharma", "David Jebb"]

def grep_chain_headers(chain_file):
    """Throw chain header lines."""
    with open(chain_file, "r") as f:
        for line in f:
            if line.startswith("chain"):
                yield line.rstrip()

def parse_chain(chain_file):
    """Read chain file.

    For each chromosome save a list of corresponding chains
    and their genomic ranges.
    """
    # save dict {chrom: list of (chain_id, start, end)} here:
    chrom_range = defaultdict(list)
    # need chains headers only (lines starting with "chain")
    # read and process them simultaneously with yield
    header_gen = grep_chain_headers(chain_file)
    for header in header_gen:
        # parse chain headers one-by-one
        if len(header) == 0:
            continue
        # space-separated fields
        header_info = header.split()
        chrom = header_info[2]
        start = int(header_info[5])
        end = int(header_info[6])
        chain_id = header_info[12]
        chrom_range[chrom].append((chain_id, start, end))
    return chrom_range

def parse_bed(bed):
    """Read bed file.

    For each chromosome save a list of corresponding transcripts
    and their genomic ranges.
    """
    chrom_range = defaultdict(list)
    f = open(bed, "r")
    for line in f:
        # parse bed-12 formatted file
        line_info = line.split("\t")
        chrom = line_info[0]
        # fields 1 and 2 -> start and end of the transcript body
        # fields 6 & 7 - start and end of the transcript CDS
        # start = int(line_info[1])
        # end = int(line_info[2])

        # count only CDS!
        start = int(line_info[6])
        end = int(line_info[7])
        transcript_id = line_info[3]
        chrom_range[chrom].append((transcript_id, start, end))
    f.close()
    return chrom_range

def intersect(range_1, range_2):
    """Return intersection size."""
    return min(range_1[2], range_2[2]) - max(range_1[1], range_2[1])

def find_first(chains, beds):
    """Find indexes for the first intersection."""
    if len(chains) == 0 or len(beds) == 0:
        # there are no chains || beds in this chrom:
        # nothing we can dc
        return 0, 0
    if intersect(chains[0], beds[0]) > 0:
        # no need to search
        # first chain intersects first bed
        return 0, 0
    # get first transcript start and first chain end
    first_bed_start, _ = beds[0][1], beds[0][2]
    _, first_chain_end = chains[0][1], chains[0][2]

    if first_chain_end < first_bed_start:
        # we have lots of chains in the beginning not intersecting beds
        for i in range(len(chains)):
            # let's go chain-by-chain until we reach first transcript
            if intersect(chains[i], beds[0]) > 0:
                return i, 0
            elif chains[i][1] > beds[0][2]:
                return i, 1
            else:  # no intersection
                return 0, 0
    else:  # lots of beds in the beginning not covered by chains
        for i in range(len(beds)):
            # go transcript-by-transcript until we reach any chain
            if intersect(chains[i], beds[0]) > 0:
                return 0, i
            elif beds[i][1] > chains[0][2]:
                return 1, i
            else:  # no intersection
                return 0, 0

def overlap(chains, beds):
    """Return intersections for chain: bed.

    Works on pre-sorted bed tracks and chains
    located on the same chromosome."""
    # init state, find FIRST bed intersecting the FIRST chain
    chain_beds = defaultdict(list)
    chain_init, start_with = find_first(chains, beds)
    chains_num = len(chains)
    bed_len = len(beds)
    bed_num = None

    if chains_num == 0 or bed_len == 0:
        # check if there are chains and beds on this chrom
        # if there is no chains and beds: nothing to intersect
        return {}

    for i in range(chain_init, chains_num):
        FLAG = False  # was there an intersection or not?
        FIRST = True
        chain = chains[i]
        while True:
            if FIRST:  # start with previous start, first iteration
                bed_num = start_with
                FIRST = False  # guarantee that this condition works ONCE per loop
            else:  # just increase the pointer
                bed_num += 1  # to avoid inf loop

            if bed_num >= bed_len:
                break  # beds are over
            # pick the bed range
            bed = beds[bed_num]

            if chain[2] < bed[1]:  # too late
                break  # means that bed is "righter" than chain

            if intersect(chain, bed) > 0:
                if not FLAG:  # the FIRST intersection of this chain
                    start_with = bed_num  # guarantee that I will assign to starts with
                    # only the FIRST intersection (if it took place)
                    FLAG = True  # otherwise starts with will be preserved
                # save the intersection
                chain_beds[chain[0]].append(bed[0])

            else:  # we recorded all the region with intersections
                if chain[1] > bed[2]:  # too early
                    # in case like:
                    # gene A EEEEE----------------------------------------EEEEEE #
                    # gene B               EEEEEEEEEE                            #
                    # gene C                               EEEEEEEEE             #
                    # chain                                    ccccc             #
                    # at gene A I will get FLAG = True and NO intersection with gene B
                    # --> I will miss gene C in this case without this condition.
                    continue

                elif FLAG:
                    # this is not a nested gene
                    # and all intersections are saved
                    # --> proceed to the next chain
                    break
    return chain_beds

def chain_bed_intersect(chain, bed):
    """Intersect bed and chain files.

    Return chain: intersected transcripts dict and
    list of transcripts not intersected by any chain.
    """
    # get list of chrom: ranges for both
    skipped = (
        []
    )  # list of genes that were skipped because they don't intersect any chain
    # read bed and chain files for the beginning
    chain_data = parse_chain(chain)
    bed_data = parse_bed(bed)

    # we have 2 dicts: chrom: genes and chrom: chains
    # sets of chromosomes in the bed and chain files might differ
    # we are interested in the sets intersection only
    # if there is chrom Y in the bed file but not chrom Y in the chain file
    # -> for sure no chrom Y genes would be intersected by any chain
    chroms = list(set(bed_data.keys()).intersection(chain_data.keys()))
    # save transcript IDs that lie on chromosomes not found in the chain file:
    only_bed_chroms = list(set(bed_data.keys()).difference(chain_data.keys()))
    genes_rejected = [
        x[0] for x in flatten([bed_data[chr_] for chr_ in only_bed_chroms])
    ]
    for gene in genes_rejected:
        skipped.append((gene, "chromosome is not aligned"))

    if len(chroms) == 0:  # no chromosomes appear in the bed and the chain file
        # for sure there is something wrong with the input data
        sys.exit("Error! No common chromosomes between the bed and chain files found!")
    chain_bed_dict = {}  # dict for result

    # main loop
    for chrom in chroms:
        # go chromosome-by-chromosome
        # of course a transcript on the chromosome 1 will never intersect
        # a chain on the chromosome 2

        # algorithm is a bit tricky and takes O(NlogN)
        # pre-sort genes and chains
        bed_ranges = sorted(bed_data[chrom], key=lambda x: x[1])
        gene_at_chrom = set(x[0] for x in bed_ranges)
        chain_ranges = sorted(chain_data[chrom], key=lambda x: x[1])
        # the intersection itself happens there:
        chrom_chain_beds = overlap(chain_ranges, bed_ranges)
        # get genes that are not intersected with any chain
        genes_in = set(flatten(v for v in chrom_chain_beds.values()))
        genes_out = gene_at_chrom.difference(genes_in)
        for gene in genes_out:
            skipped.append((gene, "no intersecting chains"))
        # add results to the main dict:
        chain_bed_dict.update(chrom_chain_beds)
    return chain_bed_dict, skipped

def save(dct, output="stdout"):
    """Save output in the file given."""
    f = open(output, "w") if output != "stdout" else sys.stdout
    for k, v in dct.items():
        f.write("{0}\t{1}\n".format(k, ",".join(v) + ","))
    f.close()

if __name__ == "__main__":
    try:  # read args
        chain_file_arg = sys.argv[1]
        bed_file_arg = sys.argv[2]
    except IndexError:
        sys.stderr.write("Usage: {} [chain_file] [bed file]\n".format(sys.argv[0]))
        sys.stderr.write("Output goes to stdout.\n")
        sys.exit(0)
    chain_bed_dict_result, _ = chain_bed_intersect(chain_file_arg, bed_file_arg)
    save(chain_bed_dict_result)


import csv, json
from collections import Counter

in_path1 = 'player_data.csv'
json_path = 'player_data.json'

# combine
with open(in_path1) as csvFile:
    csvReader = csv.DictReader(csvFile)
    in_data1 = {i:dict(row) for i, row in enumerate(csvReader)}

with open(json_path, 'w') as jsonFile:
    jsonFile.write(json.dumps(in_data1, indent=4))


import copy
from typing import Callable, Dict, Optional

import acme
from acme import specs
from acme.agents.tf.d4pg import agent
from acme.tf import savers as tf2_savers
from acme.utils import counting
from acme.utils import loggers
from acme.utils import lp_utils
import dm_env
import launchpad as lp
import reverb
import sonnet as snt
import tensorflow as tf

# Valid values of the "accelerator" argument.
_ACCELERATORS = ('CPU', 'GPU', 'TPU')

class DistributedD4PG:
  """Program definition for D4PG."""

  def __init__(
      self,
      environment_factory: Callable[[bool], dm_env.Environment],
      network_factory: Callable[[specs.BoundedArray], Dict[str, snt.Module]],
      accelerator: Optional[str] = None,
      num_actors: int = 1,
      num_caches: int = 0,
      environment_spec: Optional[specs.EnvironmentSpec] = None,
      batch_size: int = 256,
      prefetch_size: int = 4,
      min_replay_size: int = 1000,
      max_replay_size: int = 1000000,
      samples_per_insert: Optional[float] = 32.0,
      n_step: int = 5,
      sigma: float = 0.3,
      clipping: bool = True,
      discount: float = 0.99,
      policy_optimizer: Optional[snt.Optimizer] = None,
      critic_optimizer: Optional[snt.Optimizer] = None,
      target_update_period: int = 100,
      variable_update_period: int = 1000,
      max_actor_steps: Optional[int] = None,
      log_every: float = 10.0,
  ):

    if accelerator is not None and accelerator not in _ACCELERATORS:
      raise ValueError(f'Accelerator must be one of {_ACCELERATORS}, '
                       f'not "{accelerator}".')

    if not environment_spec:
      environment_spec = specs.make_environment_spec(environment_factory(False))

    # TODO(mwhoffman): Make network_factory directly return the struct.
    # TODO(mwhoffman): Make the factory take the entire spec.
    def wrapped_network_factory(action_spec):
      networks_dict = network_factory(action_spec)
      networks = agent.D4PGNetworks(
          policy_network=networks_dict.get('policy'),
          critic_network=networks_dict.get('critic'),
          observation_network=networks_dict.get('observation', tf.identity))
      return networks

    self._environment_factory = environment_factory
    self._network_factory = wrapped_network_factory
    self._environment_spec = environment_spec
    self._sigma = sigma
    self._num_actors = num_actors
    self._num_caches = num_caches
    self._max_actor_steps = max_actor_steps
    self._log_every = log_every
    self._accelerator = accelerator
    self._variable_update_period = variable_update_period

    self._builder = agent.D4PGBuilder(
        # TODO(mwhoffman): pass the config dataclass in directly.
        # TODO(mwhoffman): use the limiter rather than the workaround below.
        agent.D4PGConfig(
            accelerator=accelerator,
            discount=discount,
            batch_size=batch_size,
            prefetch_size=prefetch_size,
            target_update_period=target_update_period,
            variable_update_period=variable_update_period,
            policy_optimizer=policy_optimizer,
            critic_optimizer=critic_optimizer,
            min_replay_size=min_replay_size,
            max_replay_size=max_replay_size,
            samples_per_insert=samples_per_insert,
            n_step=n_step,
            sigma=sigma,
            clipping=clipping,
        ))

  def replay(self):
    """The replay storage."""
    return self._builder.make_replay_tables(self._environment_spec)

  def counter(self):
    return tf2_savers.CheckpointingRunner(counting.Counter(),
                                          time_delta_minutes=1,
                                          subdirectory='counter')

  def coordinator(self, counter: counting.Counter):
    return lp_utils.StepsLimiter(counter, self._max_actor_steps)

  def learner(
      self,
      replay: reverb.Client,
      counter: counting.Counter,
  ):
    """The Learning part of the agent."""

    # If we are running on multiple accelerator devices, this replicates
    # weights and updates across devices.
    replicator = agent.get_replicator(self._accelerator)

    with replicator.scope():
      # Create the networks to optimize (online) and target networks.
      online_networks = self._network_factory(self._environment_spec.actions)
      target_networks = copy.deepcopy(online_networks)

      # Initialize the networks.
      online_networks.init(self._environment_spec)
      target_networks.init(self._environment_spec)

    dataset = self._builder.make_dataset_iterator(replay)

    counter = counting.Counter(counter, 'learner')
    logger = loggers.make_default_logger(
        'learner', time_delta=self._log_every, steps_key='learner_steps')

    return self._builder.make_learner(
        networks=(online_networks, target_networks),
        dataset=dataset,
        counter=counter,
        logger=logger,
        checkpoint=True,
    )

  def actor(
      self,
      replay: reverb.Client,
      variable_source: acme.VariableSource,
      counter: counting.Counter,
  ) -> acme.EnvironmentLoop:
    """The actor process."""

    # Create the behavior policy.
    networks = self._network_factory(self._environment_spec.actions)
    networks.init(self._environment_spec)
    policy_network = networks.make_policy(
        environment_spec=self._environment_spec,
        sigma=self._sigma,
    )

    # Create the agent.
    actor = self._builder.make_actor(
        policy_network=policy_network,
        adder=self._builder.make_adder(replay),
        variable_source=variable_source,
    )

    # Create the environment.
    environment = self._environment_factory(False)

    # Create logger and counter; actors will not spam bigtable.
    counter = counting.Counter(counter, 'actor')
    logger = loggers.make_default_logger(
        'actor',
        save_data=False,
        time_delta=self._log_every,
        steps_key='actor_steps')

    # Create the loop to connect environment and agent.
    return acme.EnvironmentLoop(environment, actor, counter, logger)

  def evaluator(
      self,
      variable_source: acme.VariableSource,
      counter: counting.Counter,
      logger: Optional[loggers.Logger] = None,
  ):
    """The evaluation process."""

    # Create the behavior policy.
    networks = self._network_factory(self._environment_spec.actions)
    networks.init(self._environment_spec)
    policy_network = networks.make_policy(self._environment_spec)

    # Create the agent.
    actor = self._builder.make_actor(
        policy_network=policy_network,
        variable_source=variable_source,
    )

    # Make the environment.
    environment = self._environment_factory(True)

    # Create logger and counter.
    counter = counting.Counter(counter, 'evaluator')
    logger = logger or loggers.make_default_logger(
        'evaluator',
        time_delta=self._log_every,
        steps_key='evaluator_steps',
    )

    # Create the run loop and return it.
    return acme.EnvironmentLoop(environment, actor, counter, logger)

  def build(self, name='d4pg'):
    """Build the distributed agent topology."""
    program = lp.Program(name=name)

    with program.group('replay'):
      replay = program.add_node(lp.ReverbNode(self.replay))

    with program.group('counter'):
      counter = program.add_node(lp.CourierNode(self.counter))

    if self._max_actor_steps:
      with program.group('coordinator'):
        _ = program.add_node(lp.CourierNode(self.coordinator, counter))

    with program.group('learner'):
      learner = program.add_node(lp.CourierNode(self.learner, replay, counter))

    with program.group('evaluator'):
      program.add_node(lp.CourierNode(self.evaluator, learner, counter))

    if not self._num_caches:
      # Use our learner as a single variable source.
      sources = [learner]
    else:
      with program.group('cacher'):
        # Create a set of learner caches.
        sources = []
        for _ in range(self._num_caches):
          cacher = program.add_node(
              lp.CacherNode(
                  learner, refresh_interval_ms=2000, stale_after_ms=4000))
          sources.append(cacher)

    with program.group('actor'):
      # Add actors which pull round-robin from our variable sources.
      for actor_id in range(self._num_actors):
        source = sources[actor_id % len(sources)]
        program.add_node(lp.CourierNode(self.actor, replay, source, counter))

    return program


import sys
sys.stdin = open('room.txt', 'r')
from collections import deque

def check(y, x, lv, start):
    qu = deque([[y, x, lv, start]])
    while qu:
        r, c, nlv, st = qu.popleft()
        for i in range(4):
            ny, nx = r + dy[i], c + dx[i]
            if 0 <= ny < N and 0 <= nx < N:
                if MAP[r][c] + 1 == MAP[ny][nx]:
                    qu.append([ny, nx, nlv+1, start])
    return (nlv, st)

dy = [-1, 0, 1, 0]
dx = [0, 1, 0, -1]

for tc in range(int(input())):
    N = int(input())
    MAP = [list(map(int, input().split())) for _ in range(N)]
    path = 0
    st = 21e8
    for i in range(N):
        for j in range(N):
            res = check(i, j, 1, MAP[i][j])
            if res[0] == path:
                st = min(st, res[1])
            elif res[0] > path:
                path = max(path, res[0])
                st = res[1]
    print('#{} {} {}'.format(tc+1, st, path))


from rdflib import Graph, URIRef, Literal, BNode
from rdflib.namespace import FOAF, RDF

g = Graph()
g.bind("foaf", FOAF)

# Se añaden los datos
bob = URIRef("http://example.org/people/Bob")
g.add((bob, RDF.type, FOAF.Person))
g.add((bob, FOAF.name, Literal("Bob")))
g.add((bob, FOAF.age, Literal(38)))

# Para obtener un valor, se usa 'value'
print(g.value(bob, FOAF.age))

# Para cambiar un valor, se usa 'set'
g.set((bob, FOAF.age, Literal(39)))
print(g.value(bob, FOAF.age))