import numpy as np
import os
import shutil
import torch
from torch import nn
from torch.autograd import Variable
from torch.nn.parameter import Parameter
from torch.nn import init
from memory import Memory

def init_w_b(layer):
    init.xavier_normal(layer.weight)
    init.constant(layer.bias, 0.1)

class Actor(nn.Module):
    def __init__(self, s_dim, a_dim, a_bound, name):
        super(Actor, self).__init__()
        self.name = name
        self.a_bound = a_bound
        layers = [256, 256]
        self.fc1 = nn.Linear(s_dim, layers[0])
        self.fc2 = nn.Linear(layers[0], layers[1])
        self.out = nn.Linear(layers[1], a_dim)
        [init_w_b(l) for l in [self.fc1, self.fc2, self.out]]
        self.ac1 = nn.ReLU()
        self.tanh = nn.Tanh()
        self.scale = (self.a_bound[1] - self.a_bound[0]) / 2
        self.shift = self.a_bound[1] - self.scale
        if '_' in name:     # set target net to not bp
            self.eval()

    def forward(self, s):
        net = self.ac1(self.fc1(s))
        net = self.ac1(self.fc2(net))
        net = self.tanh(self.out(net))
        actions = net * self.scale + self.shift
        return actions

class Critic(nn.Module):
    def __init__(self, s_dim, a_dim, name):
        super(Critic, self).__init__()
        self.name = name
        layers = [256, 256]
        self.w1s = Parameter(torch.Tensor(s_dim, layers[0]))
        self.w1a = Parameter(torch.Tensor(a_dim, layers[0]))
        [w.data.normal_(0.0, 0.01) for w in [self.w1a, self.w1s]]
        self.b1 = Parameter(torch.zeros(1, layers[0])+0.1)
        self.fc2 = nn.Linear(layers[0], layers[1])
        self.out = nn.Linear(layers[1], 1)
        [init_w_b(l) for l in [self.fc2, self.out]]
        self.ac1 = nn.ReLU()
        if '_' in name:     # set target net to not bp
            self.eval()

    def forward(self, s, a):
        w1x = torch.mm(a, self.w1a) + torch.mm(s, self.w1s)
        net = self.ac1(w1x + self.b1.expand_as(w1x))
        net = self.ac1(self.fc2(net))
        s_value = self.out(net)
        return s_value

class DDPG(object):
    def __init__(self,
                 s_dim, a_dim, a_bound,
                 a_lr=0.001,c_lr=0.001,
                 tau=0.001, gamma=0.9,
                 memory_capacity=5000, batch_size=64,
                 train={'train': True, 'save_iter': None, 'load_point': -1},
                 model_dir='./torch_models',
                 ):

        self.s_dim, self.a_dim, self.a_bound = s_dim, a_dim, a_bound
        self.learn_counter = 0
        self.tau = tau
        self.gamma = gamma
        self.model_dir = model_dir
        self.batch_size = batch_size
        self.train_ = train

        self.memory_capacity = memory_capacity
        self.memory = np.empty(memory_capacity,
                               dtype=[('s', np.float32, (s_dim,)), ('a', np.float32, (a_dim,)),
                                      ('r', np.float32, (1,)), ('s_', np.float32, (s_dim,))])
        self.pointer = 0
        self.update_times = 10
        self.batch_holder = np.empty(self.batch_size*self.update_times,
                               dtype=[('s', np.float32, (s_dim,)), ('a', np.float32, (a_dim,)),
                                      ('r', np.float32, (1,)), ('s_', np.float32, (s_dim,))])

        self.anet, self.anet_ = Actor(s_dim, a_dim, a_bound, 'anet'), Actor(s_dim, a_dim, a_bound, 'anet_')
        self.cnet, self.cnet_ = Critic(s_dim, a_dim, 'cnet'), Critic(s_dim, a_dim, 'cnet_')
        self.all_nets = [self.anet, self.anet_, self.cnet, self.cnet_]
        if not self.train_['train']:
            for net in self.all_nets:
                net.load_state_dict(torch.load('./torch_models/params%i_%s.pkl' % (self.train_['load_point'], net.name)))

        self.aopt = torch.optim.Adam(self.anet.parameters(), lr=a_lr)
        self.copt = torch.optim.Adam(self.cnet.parameters(), lr=c_lr)
        self.closs_func = torch.nn.MSELoss()

    def choose_action(self, s):
        s = Variable(torch.from_numpy(s).float())
        return self.anet(s).data.numpy()

    def learn(self, lock=None):
        # hard replacement
        self._soft_rep_target()
        self._check_save()

        indices = np.random.randint(self.memory_capacity, size=self.batch_size*self.update_times)
        if lock is not None: lock.acquire()
        np.take(self.memory, indices, axis=0, out=self.batch_holder)
        if lock is not None: lock.release()

        s, a, r, s_ = self.batch_holder['s'], self.batch_holder['a'], self.batch_holder['r'], self.batch_holder['s_']
        Vs, Va, Vr, Vs_ = Variable(torch.from_numpy(s)), Variable(torch.from_numpy(a), requires_grad=True), \
                          Variable(torch.from_numpy(r)), Variable(torch.from_numpy(s_))
        for ut in range(self.update_times):
            self.learn_counter += 1
            Vbs, Vba, Vbr, Vbs_ = Vs[ut * self.batch_size: (ut + 1) * self.batch_size], \
                                  Va[ut * self.batch_size: (ut + 1) * self.batch_size], \
                                  Vr[ut * self.batch_size: (ut + 1) * self.batch_size], \
                                  Vs_[ut * self.batch_size: (ut + 1) * self.batch_size]

            target_q = Vbr + self.gamma * self.cnet_(Vbs_, self.anet_(Vbs_)).detach()    # not train
            c_loss = self.closs_func(self.cnet(Vbs, Vba), target_q)
            self.copt.zero_grad()
            c_loss.backward()
            self.copt.step()

            policy_loss = -self.cnet(Vbs, self.anet(Vbs)).mean()
            self.aopt.zero_grad()
            policy_loss.backward()
            self.aopt.step()

    def threadlearn(self, stop_event, lock=None):
        while not stop_event.is_set():
            self.learn(lock)

    def store_transition(self, s, a, r, s_):
        for item in [s,a,r,s_]:
            assert item.dtype == np.float32
        if a.ndim < 2:
            a = a[:, None]
        if r.ndim < 2:
            r = r[:, None]
        s = np.ascontiguousarray(s)
        s_ = np.ascontiguousarray(s_)
        p_ = self.pointer + r.size
        if p_ <= self.memory_capacity:
            self.memory['s'][self.pointer:p_] = s
            self.memory['a'][self.pointer:p_] = a
            self.memory['r'][self.pointer:p_] = r
            self.memory['s_'][self.pointer:p_] = s_
        else:
            p_ = p_ % self.memory_capacity
            self.memory['s'][self.pointer:] = s[:r.size - p_]
            self.memory['a'][self.pointer:] = a[:r.size - p_]
            self.memory['r'][self.pointer:] = r[:r.size - p_]
            self.memory['s_'][self.pointer:] = s_[:r.size - p_]
            self.memory['s'][:p_] = s[-p_:]
            self.memory['a'][:p_] = a[-p_:]
            self.memory['r'][:p_] = r[-p_:]
            self.memory['s_'][:p_] = s_[-p_:]
        self.pointer = p_

    def _check_save(self):
        if self.train_['save_iter'] is not None:
            if self.learn_counter % self.train_['save_iter'] == 0:
                self.save()
                print('\nSaved model for point=%i\n' % self.learn_counter)

    def save(self):
        if os.path.isdir(self.model_dir):
            shutil.rmtree(self.model_dir)
        os.mkdir(self.model_dir)
        for net in self.all_nets:
            torch.save(net.state_dict(), './torch_models/params%i_%s.pkl' % (self.learn_counter, net.name))  # save only the parameters

    def _soft_rep_target(self):
        for (aname, aparam), (cname, cparam) in zip(self.anet.state_dict().items(), self.cnet.state_dict().items()):
            self.anet_.state_dict()[aname].copy_((1-self.tau)*self.anet_.state_dict()[aname] + self.tau*aparam)
            self.cnet_.state_dict()[cname].copy_((1-self.tau)*self.cnet_.state_dict()[cname] + self.tau*cparam)

class DDPGPrioritizedReplay(DDPG):

    def __init__(self,
                 s_dim, a_dim, a_bound,
                 a_lr=0.001, c_lr=0.001,
                 tau=0.001, gamma=0.9,
                 memory_capacity=5000, batch_size=64,
                 train={'train': True, 'save_iter': None, 'load_point': -1},
                 model_dir='./model',
                 ):
        super(DDPGPrioritizedReplay, self).__init__(
            s_dim=s_dim, a_dim=a_dim, a_bound=a_bound,
            a_lr=a_lr, c_lr=c_lr,
            tau=tau, gamma=gamma,
            memory_capacity=memory_capacity, batch_size=batch_size,
            train=train, model_dir=model_dir,)
        self.memory = Memory(capacity=memory_capacity, batch_size=batch_size, s_dim=s_dim, a_dim=a_dim)

    def learn(self, lock=None):
        # hard replacement
        self._soft_rep_target()
        self._check_save()

        for _ in range(self.update_times):
            self.learn_counter += 1
            if lock is not None: lock.acquire()
            tree_idx, bt, ISWeights = self.memory.sample()

            bs, ba, br, bs_ = bt['s'], bt['a'], bt['r'], bt['s_']
            Vbs, Vba, Vbr, Vbs_, VISW = \
                Variable(torch.from_numpy(bs).float()), Variable(torch.from_numpy(ba).float()), \
                Variable(torch.from_numpy(br).float()), Variable(torch.from_numpy(bs_).float()),\
                Variable(torch.from_numpy(ISWeights).float())

            target_q = Vbr + self.gamma * self.cnet_(Vbs_, self.anet_(Vbs_)).detach()  # not train
            td_errors = self.cnet(Vbs, Vba) - target_q

            # update priority
            abs_errors = torch.abs(td_errors).data.numpy()
            self.memory.batch_update(tree_idx, abs_errors)
            if lock is not None: lock.release()

            c_loss = torch.mean(VISW * torch.pow(td_errors, 2))
            self.copt.zero_grad()
            c_loss.backward()
            self.copt.step()

            policy_loss = -self.cnet(Vbs, self.anet(Vbs)).mean()
            self.aopt.zero_grad()
            policy_loss.backward()
            self.aopt.step()

    def store_transition(self, s, a, r, s_):
        if a.ndim < 2:
            a = a[:, None]
        if r.ndim < 2:
            r = r[:, None]
        s = np.ascontiguousarray(s)
        s_ = np.ascontiguousarray(s_)
        self.memory.store(s, a, r, s_)

if __name__ == '__main__':
    import gym

    MAX_EPISODES = 70
    MAX_EP_STEPS = 400
    LR_A = 0.001  # learning rate for actor
    LR_C = 0.001  # learning rate for critic
    GAMMA = 0.9  # reward discount
    TAU = 0.01  # Soft update for target param, but this is computationally expansive
    # so we use replace_iter instead
    REPLACE_ITER_A = 500
    REPLACE_ITER_C = 300
    MEMORY_CAPACITY = 7000
    BATCH_SIZE = 32

    RENDER = False
    OUTPUT_GRAPH = True
    ENV_NAME = 'Pendulum-v0'
    var = 3
    env = gym.make(ENV_NAME)
    env = env.unwrapped
    env.seed(1)

    state_dim = env.observation_space.shape[0]
    action_dim = env.action_space.shape[0]
    action_bound = [-2, 2]
    ddpg = DDPG(state_dim, action_dim, action_bound, LR_A, REPLACE_ITER_A,
                LR_C, REPLACE_ITER_C, GAMMA, MEMORY_CAPACITY, BATCH_SIZE)
    counter = 0
    for i in range(MAX_EPISODES):
        s = env.reset()
        ep_reward = 0

        for j in range(MAX_EP_STEPS):

            if RENDER:
                env.render()

            # Add exploration noise
            a = ddpg.choose_action(s[np.newaxis, :])[0]
            a = np.clip(np.random.normal(a, var), -2, 2)  # add randomness to action selection for exploration
            s_, r, done, info = env.step(a)

            ddpg.store_transition(s[np.newaxis, :], a, np.array([r / 10]), s_[np.newaxis, :])

            if counter >= MEMORY_CAPACITY:
                var *= .9995  # decay the action randomness
                ddpg.learn()

            counter += 1
            s = s_
            ep_reward += r

            if j == MAX_EP_STEPS - 1:
                print('Episode:', i, ' Reward: %i' % int(ep_reward), 'Explore: %.2f' % var, )
                if ep_reward > -1000:
                    RENDER = True
                break


import sys
from io import StringIO
import unittest

class TestClass(unittest.TestCase):
    def assertIO(self, input, output):
        stdout, stdin = sys.stdout, sys.stdin
        sys.stdout, sys.stdin = StringIO(), StringIO(input)
        resolve()
        sys.stdout.seek(0)
        out = sys.stdout.read()[:-1]
        sys.stdout, sys.stdin = stdout, stdin
        self.assertEqual(out, output)

#     def test_Sample_Input_1(self):
#         input = """3
# <><
# 3 8 6 10"""
#         output = """2
# 1 5 4 7
# 2 3 2 3"""
#         self.assertIO(input, output)

    def test_Sample_Input_1(self):
        input = """3
<><
3 8 6 10"""
        output = """2
2 4 3 5
1 4 3 5"""
        self.assertIO(input, output)

    def test_Sample_Input_2(self):
        input = """1
<
3 4"""
        output = """1
3 4"""
        self.assertIO(input, output)

    def test_Sample_Input_3(self):
        input = """1
<
3 6"""
        output = """3
1 2
1 2
1 2"""
        self.assertIO(input, output)

def resolve():
  inf = 10**18+1
  # 差が分割数より大きいと良い非不整数列にならない。
  # 差の絶対値の最小値 K で Ai を分割して、余りを 1 ずつできるだけ割り振っていく。
  # 差の分布次第で破綻しそうな気がするけど、差の最小値を使って分割しているので、矛盾が発生しない (はず)
  inf = 10**18+1
  N = int(input())
  S = list(input())
  A = [int(x) for x in input().split(" ")]

  K = inf
  for i in range(N):
    K = min(K, abs(A[i+1]-A[i]))

  B = [[0]*(N+1) for _ in range(K)]
  for i in range(N+1):
    for k in range(K):
      B[k][i] = A[i]//K
      if A[i]%K > k: B[k][i]+=1

  print(K)
  for b in B:
    print(*b, sep=" ")

import sys
if sys.argv[-1] == './Main.py':
  resolve()

if __name__ == "__main__":
    unittest.main()


# This file is part of django-ca (https://github.com/mathiasertl/django-ca).
#
# django-ca is free software: you can redistribute it and/or modify it under the terms of the GNU General
# Public License as published by the Free Software Foundation, either version 3 of the License, or (at your
# option) any later version.
#
# django-ca is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the
# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
# for more details.
#
# You should have received a copy of the GNU General Public License along with django-ca. If not, see
# <http://www.gnu.org/licenses/>.

"""validation module to validate license headers."""

import difflib
import os
import textwrap
from pathlib import Path
from typing import Union

from devscripts.commands import CommandError

try:
    import tomllib
except ImportError:  # pragma: py<3.11
    import tomli as tomllib  # type: ignore[no-redef]

LICENSE_HEADER = """This file is part of django-ca (https://github.com/mathiasertl/django-ca).

django-ca is free software: you can redistribute it and/or modify it under the terms of the GNU General
Public License as published by the Free Software Foundation, either version 3 of the License, or (at your
option) any later version.

django-ca is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the
implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.

You should have received a copy of the GNU General Public License along with django-ca. If not, see
<http://www.gnu.org/licenses/>."""

PYTHON_BIN_HEADER = """#!/usr/bin/env python3
#
"""

PYTHON_LICENSE_HEADER = textwrap.indent(LICENSE_HEADER, "# ").replace("\n\n", "\n#\n")
PYTHON_READ_LENGTH = len(PYTHON_BIN_HEADER) + len(LICENSE_HEADER) + 64

def handle_python_file(path: Union[str, "os.PathLike[str]"], script: bool) -> int:
    """Check the license header for a Python file."""
    expected_header = PYTHON_LICENSE_HEADER
    if script is True:
        expected_header = PYTHON_BIN_HEADER + PYTHON_LICENSE_HEADER

    with open(path, encoding="utf-8") as stream:
        actual_header = stream.read(PYTHON_READ_LENGTH)

        # skip empty files
        if not actual_header:
            return 0

        if not actual_header.startswith(expected_header):
            actual_lines = actual_header.splitlines()
            expected_lines = expected_header.splitlines()

            diff = difflib.unified_diff(
                actual_lines, expected_lines, fromfile=str(path) + ".orig", tofile=str(path), lineterm=""
            )
            print("\n".join(diff))
            return 1
    return 0

def validate() -> None:
    """Main validation function."""
    errors = 0
    with open("pyproject.toml", "rb") as stream:
        config = tomllib.load(stream)

    standalone_scripts = config["django-ca"]["validation"]["standalone-scripts"]
    excludes = config["django-ca"]["validation"]["excludes"]

    for directory in ["ca", "docs/source", "devscripts"]:
        for path in sorted(Path(directory).glob("**/*.py")):
            if not any(path.match(exclude) for exclude in excludes):
                errors += handle_python_file(path, script=str(path) in standalone_scripts)

    if errors != 0:
        raise CommandError(f"{errors} inconsistent license headers found.")


import datetime
import os
import pytz

from flask import Flask
from flask_sqlalchemy import SQLAlchemy

app = Flask(__name__)
uri = os.environ.get("DATABASE_URL")
app.config["SQLALCHEMY_DATABASE_URI"] = uri
app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False

db = SQLAlchemy(app)

class Appointment(db.Model):
    __tablename__ = "appointments"
    id = db.Column(db.BigInteger, primary_key=True, nullable=False, autoincrement=True)
    time = db.Column(db.DateTime())
    location = db.Column(db.String(length=256), nullable=True)
    name = db.Column(db.String(length=256), nullable=True)
    position = db.Column(db.String(length=256), nullable=True)
    email = db.Column(db.String(length=256), nullable=True)
    category = db.Column(db.String(length=256), nullable=True)
    phone = db.Column(db.String(length=256), nullable=True)
    filled = db.Column(db.Boolean, default=False)
    visible = db.Column(db.Boolean, default=True)

    def add_slot(month, day, hour, minute, location=None):
        time = datetime.datetime(
            month=month, day=day, year=2018, hour=hour, minute=minute,
        )
        appt = Appointment(time=time, name=None, position=None, filled=False, location=location)
        db.session.add(appt)
        db.session.commit()

    def remove_slots(month, day):
        for appt in Appointment.query.all():
            if appt.time.month == month and appt.time.day == day:
                appt.visible = False
        db.session.commit()


import yaml

def is_yaml_file_corrupted(file_path):
    try:
        with open(file_path, 'r') as yaml_file:
            yaml.safe_load(yaml_file)
        return False  # Parsing successful, file is not corrupted
    except yaml.YAMLError as e:
        print(f"YAML parsing error: {str(e)}")
        return True  # Parsing failed, file is corrupted

# Replace 'path/to/your/file.yaml' with the actual file path you want to check
file_path = 'personinfo.yaml'

if is_yaml_file_corrupted(file_path):
    print(f"The YAML file at {file_path} is corrupted.")
else:
    print(f"The YAML file at {file_path} is not corrupted.")


# -*- coding: utf-8 -*-
"""
Created on Tue Oct 26 15:45:18 2021

@author: icruicks
"""
from abc import ABC, abstractmethod
import requests, urllib3, re, os, logging, time, random, string
import concurrent.futures
import pandas as pd
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.common.exceptions import WebDriverException
from bs4 import BeautifulSoup
from newsplease import NewsPlease
from newspaper import Article
try:
    from unshortenit import UnshortenIt
    unshortener_available = True
except:
    unshortener_available = False
from urllib.parse import urljoin, urlunparse, urlparse, parse_qs

'''
Set global variables and important lists of stuff like links to unshorten, or
those to keep query terms on, etc.
'''

MAX_THREADS = 20

LINK_SHORTENERS =set(['trib.al', 'bit.ly','www.bit.ly','tinyurl','ow.ly','buff.ly',
                           'rebrand.ly', 'dlvr.it','sco.lt', 'shar.es', 'spr.ly',
                           'zpr.io', 'zurl.co', 'tinyurl.com', 'ht.ly', 'youtu.be',
                           't.ly', 'snip.ly', 'qoo.ly', 'loom.ly', 'invst.ly',
                           'hubs.ly', 'gates.ly', 'frost.ly', 'fcld.ly', 'cutt.ly',
                           'all.be', 'amzn.to', 'goo.gl', 'is.gd', 'bit.do', 'mcaf.ee',
                           'shorte.st', 'bc.vc', 'lnkd.in', 't.co', 'ift.tt', 'flip.it',
                           'reut.rs', 'nyti.ms', 'chng.it', 'cnn.it', 'cnb.cx', 'mol.im',
                           'paper.li', 'toi.in', 'flip.it', 'hill.cm', 'bbc.in',
                           'ti.me', 'politi.co', 'aje.io', 'gizmo.do', 'tiny.iavian.net',
                           'w.wiki', 'w-j.com', 'wp.lnjmp.com', 'hann.it', 'feedproxy.google.com',
                           'tiny.iavian.com'])

FALSE_LINK_SHORTENERS = set( ["t.me"

                              ])

KEEP_QUERY_TERMS = ['www.youtube.com', 'youtube.com', 'm.youtube.com', 'youtu.be',
                    'None','m.facebook.com', 'facebook.com','theresistance.video']

'''
Define the top-level abstract class for the various URL data utilities. The general
format consist in setting the parameters of the collection, the actual code for
the collection itself, and multi-threading the actual collection piece.
'''

class internet_data_collection(ABC):

    @abstractmethod
    def retrieve_from_url(self, url):
        pass

    def retreive_from_urls(self, list_of_urls):
        threads = MAX_THREADS
        with concurrent.futures.ThreadPoolExecutor(max_workers=threads) as executor:
            collection_results = executor.map(self.retrieve_from_url, list_of_urls)

        return pd.DataFrame(collection_results)

'''
Define module-level functions
'''

def get_domain(url):
    '''
    Gets the high-level domain of a URL

    Parameters
    ----------
    url : string
        URL of a website

    Returns
    -------
    domain : string
        High-level domain of that website
    '''

    if url !='None':
        domain = urlparse(url).netloc.lower()
        if domain[0:4] =='www.':
            domain = domain[4:]
    else:
        domain = 'None'
    return domain

def map_mobile_to_original(url):
    """
    check for whether a url is a mobile url, and then map it back to
    the orginal website.

    Parameters
    ----------
    url : string
        url of website

    Returns
    -------
    new_url : str
        url that is not a mobile type
    mobile : bool
        wether the url was a mobile type

    """
    u = urlparse(url)
    if u.path.lower()[0:2] == 'm.':
        new_url = urlunparse( u._replace(path = u.path.lower()[2:]))
        mobile= True
    elif u.netloc.lower()[0:2] == 'm.':
        new_url = urlunparse(u._replace(netloc = u.netloc.lower()[2:]))
        mobile = True
    else:
        new_url = url
        mobile = False
    return new_url, mobile

def remove_query_terms(url, keep_query_terms=KEEP_QUERY_TERMS):
    """
    check for whether a url has query terms ('?'), and if not a website
    that uniuely identifies content by query terms, remove them

    Parameters
    ----------
    url : string
        URL of a website
    keep_query_terms : set of str
        set or other iterable of domains that
        need to keep query terms NOTE: uses globally variable as default.

    Returns
    -------
    new_url : str
        url without query terms

    """
    query = urlparse(url)
    if (query.path.lower() not in keep_query_terms) & (query.netloc.lower() not in keep_query_terms):
        new_url = urljoin(url, query.path)
    elif query.hostname in ('www.youtube.com', 'youtube.com', 'm.youtube.com'):
        p = parse_qs(query.query)
        video_id = p.get('v',['None'])[0]
        new_url = query.scheme+"://"+query.netloc+query.path+"?v="+video_id
    else:
        new_url = url
    return new_url

def get_base_url(url, link_shorteners=LINK_SHORTENERS, false_link_shorteners = FALSE_LINK_SHORTENERS, num_tries=3):
    """
    check for whether a url is shortened, and then unshorten
    it if it is.

    Parameters
    ----------
    url : string
        URL of a website
    link_shorteners : set of str
        set or other iterable of link shortener domains
        to check for. NOTE: uses globally variable as default.

    Returns
    -------
    new_url : str
        url without query terms
    shortened : bool
        wether the url was a shortened url

    """

    url_netloc = urlparse(url).netloc.lower()
    if ((url_netloc in link_shorteners) | (bool(re.match(r"\.\w\w", url_netloc[-3:])))) & (url_netloc not in false_link_shorteners):
        shortened = True
        if unshortener_available:
            for i in range(num_tries):
                try:
                    unshortener = UnshortenIt(default_timeout=20)
                    base_url = unshortener.unshorten(url, unshorten_nested=True,
                                                     force=True)
                except:
                    logging.error("unable to unshorten: "+url)
                    base_url = url
                    time.sleep(random.uniform(30,120))
                else:
                    return base_url, shortened
        else:
            urllib3.disable_warnings()
            session = requests.Session()
            for i in range(num_tries):
                try:
                    resp = session.head(url, allow_redirects=True, timeout=20.0, verify=False)
                    base_url = resp.url
                except:
                    logging.error("unable to unshorten: "+url)
                    base_url = url
                    time.sleep(random.uniform(30,120))
                else:
                    return base_url, shortened
        return base_url, shortened
    else:
        shortened = False
        return url, shortened

'''
Define module-level classes that inheret from internet_data_collection
abstract class
'''

class process_urls(internet_data_collection):

    def __init__(self, unshorten=True, map_mobile=True):
        self.unshorten = unshorten
        self.map_mobile = map_mobile

    def retrieve_from_url(self, url):
        datum ={}
        datum['original_url'] = url
        if self.unshorten:
            url, datum['shortened'] = get_base_url(url)

        url = remove_query_terms(url)

        if self.map_mobile:
            url, datum['mobile'] = map_mobile_to_original(url)

        datum['domain'] = get_domain(url)
        datum['base_url'] = url
        return datum

    def retreive_from_urls(self, list_of_urls):
        threads = min(len(list_of_urls), MAX_THREADS)
        with concurrent.futures.ThreadPoolExecutor(max_workers=threads) as executor:
            collection_results = executor.map(self.retrieve_from_url, list_of_urls)

        return pd.DataFrame(collection_results)

class get_data_from_urls(internet_data_collection):

    def __init__(self, num_retries=3, path_to_chromedriver=None):
        self.num_retries = num_retries
        self.path_to_chromedriver = path_to_chromedriver
        if path_to_chromedriver is not None:
            self.use_selenium= True
        else:
            self.use_selenium = False
    '''
    def retrieve_from_url(self, url):
        datum= {'url':url}
        html = None
        logging.info("Getting text from: "+url)
        for attempt in range(self.num_retries):
            try:
                html = requests.get(url, headers={"User-Agent": "Requests"},
                                    allow_redirects=True, timeout=10).content
            except requests.exceptions.SSLError:
                try:
                    html = requests.get(url, headers={"User-Agent": "Requests"},
                                        allow_redirects=True, timeout=10, verfiy=False).content
                    break
                except:
                    break
            except requests.exceptions.RequestException:
                time.sleep(random.uniform(30,120))
            else:
                break

        if html == None:
            logging.error("Unable to get text by requests for: "+url)
            if self.use_selenium:
                try:
                    options = Options()
                    options.add_argument('--headless')
                    with webdriver.Chrome(self.path_to_chromedriver, options=options) as driver:
                        driver.set_page_load_timeout(30)
                        driver.get(url)
                        raise WebDriverException
                    html = driver.page_source
                except:
                    logging.error("Unable to get html from website: "+url)
                    datum['text']= "exception occurred with accessing url"
                    return datum
            else:
                datum['text']= "exception occurred with accessing url"
                return datum

        try:
            soup = BeautifulSoup(html, 'html.parser')
            text = ''
            for tag in soup.find_all(['p', 'text']):
                text = text + ' ' + re.sub(r'\n\s*\n', r' ', tag.get_text().strip(), flags=re.M)
                if len(text) > 3000000:
                    break

        except:
            logging.error("Unable to parse result: "+url)
            datum['text']= "exception occurred with html parsing"
        else:
            if len(text) <=500:
                datum['text'] = "unable to extract meaningful text"
            else:
                datum['text']= text

        return datum
    '''

    def retrieve_from_url(self, url):
        datum = self.retrieve_from_url_newspaper3k(url)

        if datum['text'] == "error in scraping":
            datum = self.retrieve_from_url_newsplease(url)
            if datum['text'] == "error in scraping":
                datum = self.retrieve_from_url_requests(url)

        return datum

    def retrieve_from_url_requests(self, url):
        datum= {'url':url,'image_url':None,
                'language':None, 'text':None,'title':None
               }
        html = None
        logging.info("Trying requests text scrape: "+url)
        for attempt in range(self.num_retries):
            try:
                html = requests.get(url, headers={"User-Agent": "Requests"},
                                    allow_redirects=True, timeout=10).content
            except requests.exceptions.SSLError:
                try:
                    html = requests.get(url, headers={"User-Agent": "Requests"},
                                        allow_redirects=True, timeout=10, verfiy=False).content
                    break
                except:
                    break
            except requests.exceptions.RequestException:
                time.sleep(random.uniform(30,120))
            else:
                break

        if html == None:
            logging.error("Unable to get text by requests for: "+url)
            if self.use_selenium:
                try:
                    options = Options()
                    options.add_argument('--headless')
                    with webdriver.Chrome(self.path_to_chromedriver, options=options) as driver:
                        driver.set_page_load_timeout(30)
                        driver.get(url)
                        raise WebDriverException
                    html = driver.page_source
                except:
                    logging.error("Unable to get html from website: "+url)
                    datum['text'] = None
                    return datum
            else:
                datum['text'] = None
                return datum

        try:
            soup = BeautifulSoup(html, 'html.parser')
            parsed_text = ''
            for tag in soup.find_all(['p', 'text']):
                parsed_text = parsed_text + ' ' + re.sub(r'\n\s*\n', r' ', tag.get_text().strip(), flags=re.M)
                if len(parsed_text) > 3000000:
                    break

        except:
            logging.error("Unable to parse result: "+url)
            datum['text'] = None
        else:
            if len(parsed_text) <=500:
                datum['text'] = None
            else:
                text = parsed_text

        return datum

    def retrieve_from_url_newsplease(self, url):
        datum= {'url':url}
        article = None
        logging.info("Getting text from (NewsPlease): "+url)
        for attempt in range(self.num_retries):
            try:
                article = NewsPlease.from_url(url, timeout=10).get_dict()
            except:
                logging.error("Unable to scrape: "+url)

            if article == None:
                datum['text']= "error in scraping"
                #datum['date_publish'] = None
                datum['image_url'] = None
                datum['language'] = None
                datum['text'] = None
                datum['title'] = None
                break
            else:
                if article["maintext"] == None:
                    datum['text'] = "error in scraping"
                else:
                    datum['text'] = article['maintext']
                '''
                if article['date_publish'] is not None:
                    datum['date_publish'] = article['date_publish'].strftime("%m/%d/%Y")
                else:
                    datum['date_publish'] = None
                '''
                datum['image_url'] = article['image_url']
                datum['language'] = article['language']
                datum['title'] = article["title"]
                break
        return datum

    def retrieve_from_url_newspaper3k(self, url):
        datum= {'url':url}
        article = None
        logging.info("Getting text from (newspaper3k): "+url)
        for attempt in range(self.num_retries):
            try:
                article = Article(url=url)
                article.download()
                article.parse()
            except:
                logging.error("Unable to scrape: "+url)

            if article == None:
                datum['text']= "error in scraping"
                datum['date_publish'] = None
                datum['image_url'] = None
                datum['language'] = None
                datum['text'] = None
                datum['authors'] = None
                datum['title'] = None
                break
            else:
                datum['text'] = article.text
                datum['date_publish'] = article.publish_date.strftime("%m/%d/%Y")
                datum['image_url'] = article.top_image
                datum['authors'] = article.authors
                datum['title'] = article.title
                break
        return datum

    def retreive_from_urls(self, list_of_urls):
        threads = min(MAX_THREADS, len(list_of_urls))
        with concurrent.futures.ThreadPoolExecutor(max_workers=threads) as executor:
            processed_urls =executor.map(self.retrieve_from_url, list_of_urls)

        return_df = pd.DataFrame(processed_urls)

        logging.info("succesfully created df: {} and {}".format(list_of_urls[0], list_of_urls[-1]))

        return return_df

class get_images_from_urls(internet_data_collection):

    def __init__(self, num_retries=3, path_to_chromedriver=None,
                 image_save_directory="images"):
        self.num_retries = num_retries
        self.path_to_chromedriver = path_to_chromedriver
        if path_to_chromedriver is not None:
            self.use_selenium= True
        else:
            self.use_selenium = False

        self.image_save_directory = image_save_directory
        if image_save_directory == "images":
            if image_save_directory not in os.listdir():
                try:
                    os.mkdir(image_save_directory)
                except OSError:
                    pass

    def retrieve_from_url(self, url):
        datum= {'url':url,
                'image_urls':[],
                'image_names':[]}
        html = None
        logging.info("Getting image from: "+url)
        for attempt in range(self.num_retries):
            try:
                html = requests.get(url, headers={"User-Agent": "Requests"},
                                    allow_redirects=True, timeout=10).content
            except requests.exceptions.SSLError:
                try:
                    html = requests.get(url, headers={"User-Agent": "Requests"},
                                        allow_redirects=True, timeout=10, verfiy=False).content
                    break
                except:
                    break
            except requests.exceptions.RequestException:
                time.sleep(random.uniform(30,120))
            else:
                break

        if html == None:
            logging.error("Unable to get images by requests for: "+url)
            if self.use_selenium:
                try:
                    options = Options()
                    options.add_argument('--headless')
                    with webdriver.Chrome(self.path_to_chromedriver, options=options) as driver:
                        driver.get(url)
                        driver.set_page_load_timeout(30)
                        raise WebDriverException
                    html = driver.page_source
                except:
                    logging.error("Unable to get html from website: "+url)
                    datum['image_name']= "None"
                    return datum
            else:
                datum['image_name']= "None"
                return datum

        try:
            soup = BeautifulSoup(html, 'html.parser')
            img_tags = soup.find_all('img')
            urls = [img['src'] for img in img_tags]
            datum['image_urls'] = urls
            image_filenames = [''.join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(6))+i[-4:] for i in urls]
            datum['image_names'] = image_filenames
            for i in range(len(urls)):
                image_url = urls[i]
                image_filename = image_filenames [i]
                try:
                    with open(os.path.join(self.image_save_directory, image_filename), 'wb') as f:
                        if 'http' not in url:
                            # sometimes an image source can be relative
                            # if it is provide the base url which also happens
                            # to be the site variable atm.
                            image_url = '{}/{}'.format(url, image_url)
                        response = requests.get(image_url)
                        f.write(response.content)
                except:
                    logging.error("Unable to access image: "+image_url)

        except:
            logging.error("Unable parse images from html: "+url)

        return datum

    def retreive_from_urls(self, list_of_urls):
        threads = min(MAX_THREADS, len(list_of_urls))
        with concurrent.futures.ThreadPoolExecutor(max_workers=threads) as executor:
            processed_urls =executor.map(self.retrieve_from_url, list_of_urls)

        return pd.DataFrame(processed_urls)


from api import models
from publicFunc import Response
from publicFunc import account
from django.http import JsonResponse
from publicFunc.condition_com import conditionCom
from publicFunc.play_watermark import watermark # 图片打水印
from api.forms.posters import AddForm, UpdateForm, SelectForm, posterInfoForm
from publicFunc.base64_encryption import b64decode
import json, os, requests

# token验证 海报展示模块
@account.is_token(models.Userprofile)
def posters(request):
    response = Response.ResponseObj()
    # user_id = request.GET.get('user_id')
    if request.method == "GET":
        forms_obj = SelectForm(request.GET)
        if forms_obj.is_valid():
            current_page = forms_obj.cleaned_data['current_page']
            length = forms_obj.cleaned_data['length']
            user_id = request.GET.get('user_id')

            order = request.GET.get('order', '-create_datetime')
            field_dict = {
                'id': '',
                'posters_url': '__contains',
                'posters_status': '',
                'create_datetime': '',
                'create_user_id': '',
                'create_user__name': '__contains',
            }
            q = conditionCom(request, field_dict)
            print('q -->', q)
            objs = models.Posters.objects.filter(q).order_by(order)
            count = objs.count()

            if length != 0:
                start_line = (current_page - 1) * length
                stop_line = start_line + length
                objs = objs[start_line: stop_line]

            # 返回的数据
            ret_data = []

            user_obj = models.Userprofile.objects.get(id=user_id)

            for obj in objs:
                #  将查询出来的数据 加入列表
                ret_data.append({
                    'id': obj.id,
                    'posters_url': obj.posters_url,
                    'posters_status_id': obj.posters_status,
                    'posters_status': obj.get_posters_status_display(),
                    'create_user_id': obj.create_user_id,
                    'create_user__name': obj.create_user.name,
                    'create_datetime': obj.create_datetime.strftime('%Y-%m-%d %H:%M:%S'),
                })

            posters_choices = []
            for i in models.Posters.posters_choices:
                posters_choices.append({
                    'id': i[0],
                    'name': i[1]
                })
            #  查询成功 返回200 状态码
            response.code = 200
            response.msg = '查询成功'
            response.data = {
                'ret_data': ret_data,
                'data_count': count,
                'posters_choices': posters_choices,
            }

            response.note = {
                'id': "海报ID",
                'posters_url': "海报链接",
                'posters_status_id': "海报类型ID",
                'posters_status': "海报类型名称",
                'create_datetime': "创建时间",
                'create_user_id': "创建人ID",
                'create_user__name': "创建人名字",
            }

        else:
            response.code = 402
            response.msg = "请求异常"
            response.data = json.loads(forms_obj.errors.as_json())
    return JsonResponse(response.__dict__)

# 海报增删改
# token验证
@account.is_token(models.Userprofile)
def posters_oper(request, oper_type, o_id):
    response = Response.ResponseObj()
    user_id = request.GET.get('user_id')

    if request.method == "POST":
        form_data = {
            'o_id': o_id,
            'create_user_id': user_id,
            'posters_url': request.POST.get('posters_url'),
            'posters_status': request.POST.get('posters_status', 1),
        }

        # 添加海报
        if oper_type == "add":
            forms_obj = AddForm(form_data)
            if forms_obj.is_valid():
                print("验证通过")
                obj = models.Posters.objects.create(**forms_obj.cleaned_data)
                response.code = 200
                response.msg = "添加成功"
                response.data = {'id': obj.id}
            else:
                print("验证不通过")
                response.code = 301
                response.msg = json.loads(forms_obj.errors.as_json())

        # 修改海报
        elif oper_type == "update":
            forms_obj = UpdateForm(form_data)
            if forms_obj.is_valid():
                print("验证通过")
                o_id, objs = forms_obj.cleaned_data['o_id']
                posters_url = forms_obj.cleaned_data['posters_url']
                posters_status = forms_obj.cleaned_data['posters_status']

                objs.update(posters_url=posters_url, posters_status=posters_status)
                response.code = 200
                response.msg = "修改成功"

            else:
                print("验证不通过")
                response.code = 301
                response.msg = json.loads(forms_obj.errors.as_json())

        # 删除海报
        elif oper_type == "delete":
            # 删除 ID
            objs = models.Posters.objects.filter(id=o_id)
            if objs:
                objs.delete()
                response.code = 200
                response.msg = "删除成功"
            else:
                response.code = 302
                response.msg = '删除ID不存在'

        # 修改海报信息
        # elif oper_type == 'update_poster_info':
        #     form_data = {
        #         'title': request.POST.get('title'),  # 正标题
        #         'subtitle': request.POST.get('subtitle'),  # 副标题
        #         'name': request.POST.get('name'),  # 姓名
        #         'phone': request.POST.get('phone'),  # 电话
        #         'time': request.POST.get('time'),  # 时间
        #         'place': request.POST.get('place'),  # 地点
        #     }
        #
        #     PosterObj = UpdatePosterInfoForm(form_data)
        #     if PosterObj.is_valid():
        #         objs = models.Userprofile.objects.filter(id=user_id)
        #         objs.update(posters_info=PosterObj.data)
        #         response.code = 200
        #         response.msg = '修改成功'
        #     else:
        #         response.code = 301
        #         response.msg = json.loads(PosterObj.errors.as_json())

    else:

        # 海报打水印
        if oper_type == 'play_watermark':
            posters_status = request.GET.get('posters_status')  # 水印类型
            posters_objs = models.Posters.objects.filter(id=o_id)
            if posters_objs:
                img_path = posters_objs[0].posters_url              # 海报地址

                posters_status = int(posters_status)
                if posters_status == 1:
                    obj = models.Userprofile.objects.get(id=user_id)
                    data = {
                        'img_path': img_path,
                        'heading':obj.set_avator + '?imageView2/2/w/300',
                        'name':b64decode(obj.name),
                        'phone':obj.phone_number,
                        'posters_status':posters_status,
                    }
                    watermark_objs = watermark(data)  # 实例化
                    path = watermark_objs.posters_play_watermark()
                    response.code = 200
                    response.data = {'path': path}

                else:
                    place = request.GET.get('place')                # 地址
                    time = request.GET.get('time')                  # 时间
                    fu_title = request.GET.get('subtitle')          # 副标题
                    zhu_title = request.GET.get('title')            # 主标题
                    name = request.GET.get('name')                  # 姓名
                    phone = request.GET.get('phone')                # 电话

                    data = {
                        'posters_status': posters_status,
                        'img_path': img_path,
                        'name': name,
                        'phone': phone,
                        'zhu_title': zhu_title,
                        'fu_title': fu_title,
                        'time': time,
                        'place': place,
                    }
                    form = posterInfoForm(data)
                    if form.is_valid():
                        watermark_objs = watermark(data) # 实例化
                        path = watermark_objs.posters_play_watermark()
                        response.code = 200
                        response.msg = '生成成功'
                        response.data = {'path':path}
                    else:
                        response.code = 301
                        response.msg = json.loads(form.errors.as_json())

            else:
                response.code = 301
                response.msg = '海报被删除'

        else:
            response.code = 402
            response.msg = "请求异常"

    return JsonResponse(response.__dict__)


import argparse
import pickle
import sys
import subprocess
import os
import colorama
from colorama import init,Fore, Back, Style

parser = argparse.ArgumentParser(description='Parse test description')
parser.add_argument('-avh', nargs='?',type = str, default="C:/Keil_v5/ARM/VHT", help="AVH folder")
parser.add_argument('-d', action='store_true', help="Debug log")
parser.add_argument('-n', action='store_true', help="No force rebuild")
parser.add_argument('-r', action='store_true', help="Raw results only")
parser.add_argument('-c', action='store_true', help="Display cycles (so passing test are displayed)")

args = parser.parse_args()

DEBUG=False
if args.d:
    DEBUG=True

init()

sys.path.append("..")

from TestScripts.Parser import TreeElem

ERROR_OCCURED = False

def printTitle(s):
    print("\n" + Fore.GREEN + Style.BRIGHT +  s + Style.RESET_ALL)

def printSubTitle(s):
    print("\n" + Fore.YELLOW + Style.BRIGHT + s + Style.RESET_ALL)

def printError(s):
    print("\n" + Fore.RED + Style.BRIGHT +  s + Style.RESET_ALL)

# Load Output.pickle files for the test description
def loadRoot(f):
    root = None
    with open(f,"rb") as inf:
         root=pickle.load(inf)
    return(root)

# Get test suites from the test descriptions
def getSuites(node,filterList,currentList=[]):
    if node.kind==TreeElem.SUITE:
        currentList.append(node.data["class"])
    if node.kind==TreeElem.GROUP:
       if not node.data["class"] in filterList:
          for c in node.children:
             getSuites(c,filterList,currentList)

class Result:
    def __init__(self,msg,error=False):
        self._error = error
        self._msg = msg

    @property
    def error(self):
        return self._error

    @property
    def msg(self):
        return self._msg

# Run a command and get error or result
# For the test report we don't need the stderr
# in case of error since the test report is giving
# all the details. So, there is an option to
# disable the dump of stderr
def run(*args,mustPrint=False,dumpStdErr=True):
    global ERROR_OCCURED
    global DEBUG
    try:
        if DEBUG:
            print(" ".join(args))
        result=subprocess.run(args,text=True,capture_output=True,timeout=600)
        if result.returncode !=0 :
             ERROR_OCCURED = True
             if dumpStdErr:
                return(Result(result.stderr + "\n\nSTDOUT:\n\n" + result.stdout,error=True))
             else:
                return(Result(result.stdout,error=True))

        if mustPrint:
            print(result.stdout)
        return(Result(result.stdout))
    except Exception as e:
        printError("Exception occured")
        ERROR_OCCURED = True
        return(Result(str(e),error=True))

# Configuration file for AVH core
configFiles={
    "CS310":"ARM_VHT_Corstone_310_config.txt",
    "CS300":"ARM_VHT_Corstone_300_config.txt",
    "M55":"ARM_VHT_MPS2_M55_config.txt",
    "M33_DSP_FP":"ARM_VHT_MPS2_M33_DSP_FP_config.txt",
    "M7DP":"ARM_VHT_MPS2_M7DP_config.txt",
    "M4FP":"ARM_VHT_MPS2_M4FP_config.txt",
    "M3":"ARM_VHT_MPS2_M3_config.txt",
    "M23":"ARM_VHT_MPS2_M23_config.txt",
    "M0plus":"ARM_VHT_MPS2_M0plus_config.txt",
}

# Windows executable
# (At some point this script will also support
# unix)
avhExe={
    "CS310":"VHT_Corstone_SSE-310.exe",
    "CS300":"VHT_Corstone_SSE-300_Ethos-U55.exe",
    "M55":"VHT_MPS2_Cortex-M55.exe",
    "M33_DSP_FP":"VHT_MPS2_Cortex-M33.exe",
    "M7DP":"VHT_MPS2_Cortex-M7.exe",
    "M4FP":"VHT_MPS2_Cortex-M4.exe",
    "M3":"VHT_MPS2_Cortex-M3.exe",
    "M23":"VHT_MPS2_Cortex-M23.exe",
    "M0plus":"VHT_MPS2_Cortex-M0plus.exe",
}

AVHROOT = args.avh

# Run AVH
def runAVH(build,core):
    axf="out/test/%s/Release/test.axf" % (build,)
    elf="out/test/%s/Release/test.elf" % (build,)
    app = axf
    if os.path.exists(axf):
        app = axf
    if os.path.exists(elf):
        app = elf
    config = os.path.join("configs",configFiles[core])
    avh = os.path.join(AVHROOT,avhExe[core])
    res=run(avh,"-f",config,app)
    return(res)

####################

# Test descriptions to use
tests=["../Output.pickle","../Output_f16.pickle"]
# Test group to avoid
filterList=["NNTests","ExampleTests"]
allSuites=[]

# Load all the test suite to run
for t in tests:
    root=loadRoot(t)
    suites=[]
    getSuites(root,filterList,suites)
    allSuites += [(x,t) for x in suites ]

# Test suite and output pickle needed to decode the result
#print(allSuites)

#allSuites=[
#("MFCCQ15","../Output.pickle"),
#("MFCCQ31","../Output.pickle"),
#("SupportTestsF16","../Output_f16.pickle"),
#]

#allSuites=[("ComplexTestsF32","../Output.pickle"),
#("DistanceTestsF32","../Output.pickle"),
#("UnaryTestsF32","../Output.pickle"),
#("QuaternionTestsF32","../Output.pickle"),
#("StatsTestsF32","../Output.pickle")
#]

#allSuites=[
#("TransformCF64","../Output.pickle"),
#("TransformCF32","../Output.pickle"),
#("TransformCQ31","../Output.pickle"),
#("TransformCQ15","../Output.pickle"),
#
#("TransformRF64","../Output.pickle"),
#("TransformRF32","../Output.pickle"),
#("TransformRQ31","../Output.pickle"),
#("TransformRQ15","../Output.pickle"),
#
#("MFCCF32","../Output.pickle"),
#("MFCCQ31","../Output.pickle"),
#("MFCCQ15","../Output.pickle"),
#
#("MFCCF16","../Output_f16.pickle"),
#("TransformCF16","../Output_f16.pickle"),
#("TransformRF16","../Output_f16.pickle")
#]

#allSuites=[("StatsTestsQ7","../Output.pickle")]

allSuites=[
("MISCF32","../Output.pickle"),
#("MISCQ15","../Output.pickle"),
#("MISCQ7","../Output.pickle"),
#("FIRF16","../Output_f16.pickle")
]

# Solution and build file for all
# the tests
# It is a pair : csolution target type and AVH identification
# AVH identification is used to find the executable
# and the configuration file
solutions={
    'test_ac6.csolution.yml':[
    #  ("VHT-Corstone-310","CS310"),
      ("VHT-Corstone-300","CS300"),
      ("VHT_M33","M33_DSP_FP"),
      ("VHT_M7","M7DP"),
      ("VHT_M7_UNROLLED","M7DP"),
      ("VHT_M4","M4FP"),
    #  #("VHT_M3","M3"),
    #  #("VHT_M23","M23"),
      ("VHT_M0P","M0plus")
    ],
    'test_gcc.csolution.yml':[
      #("VHT-Corstone-310","CS310"),
      #("VHT_M55","M55"),
      ##("VHT_M33","M33_DSP_FP"),
      ("VHT_M7","M7DP"),
      ("VHT_M7_UNROLLED","M7DP"),
      ("VHT_M4","M4FP"),
      ##("VHT_M3","M3"),
      ##("VHT_M23","M23"),
      ("VHT_M0P","M0plus")
    ]
}

# Override previous solutions for more restricted testing.
solutions={
    'test_ac6.csolution.yml':[
      ("VHT-Corstone-300","CS300"),
    ]
}

HTMLHEADER="""

<title>CMSIS-DSP Test summary</title>

"""

HTMLFOOTER="""

"""

# Run the tests and log the result
# in a summary.html file
with open("summary.html","w") as f:
    print(HTMLHEADER,file=f)
    for s in solutions:
        printTitle("Process solution %s" % s)
        res=run("csolution","convert","-s",s)
        if res.error:
            printError("Error csolution")
            print("
Error converting csolution %s

" % s,file=f)
            print(res.msg,file=f)
            print("
",file=f)
            continue
        print("
Solution %s
" % s,file=f)
        maxNbBuilds=len(solutions[s])
        buildNb=0
        for build,core in solutions[s]:
            buildNb = buildNb + 1
            print("
Core %s
" % build,file=f)
            printTitle("Process core %s (%d/%d)" % (build,buildNb,maxNbBuilds))
            buildFile="test.Release+%s.cprj" % build
            nb = 0
            maxNb = len(allSuites)
            for s,pickle in allSuites:
                nb = nb + 1
                printSubTitle("Process suite %s (%d/%d)" % (s,nb,maxNb))
                res=run(sys.executable,"../processTests.py","-gen","..","-p","../Patterns","-d","../Parameters","-f",pickle,"-e",s,mustPrint=True)
                if res.error:
                    printError("Error processTests")
                    print("
Error generating %s

" % s,file=f)
                    print(res.msg,file=f)
                    print("
",file=f)
                    continue
                if nb==1:
                   # -r is needed for first
                   # build when we switch
                   # between different solutions
                   # (Like one using AC6 and the other
                   # using gcc)
                   if args.n:
                      res=run("cbuild",buildFile)
                   else:
                      res=run("cbuild","-r",buildFile)
                else:
                   res=run("cbuild",buildFile)
                if res.error:
                    printError("Error cbuild")
                    print("
Error building %s

" % s,file=f)
                    print(res.msg,file=f)
                    print("
",file=f)
                    continue
                printSubTitle("Run AVH")
                res=runAVH(build,core)
                if res.error:
                    printError("Error running AVH")
                    print("
Error running %s

" % s,file=f)
                    print(res.msg,file=f)
                    print("
",file=f)
                    continue
                else:
                    with open("results.txt","w") as o:
                        print(res.msg,file=o)
                    # Dump raw result
                    if args.r:
                        print(res.msg)
                # If raw result, no post processing
                if not args.r:
                    res=run(sys.executable,"../processResult.py","-f",pickle,"-e","-ahtml","-r","results.txt",dumpStdErr=False)
                    if res.error:
                        printError("Error processResult")
                        print("
Error processing %s result

" % s,file=f)
                        print(res.msg,file=f)
                        print("
",file=f)
                        continue
                    else:
                        # When no error the section is
                        # included in final file on when
                        # cycles are requested
                        if args.c:
                           print(res.msg,file=f)
    print(HTMLFOOTER,file=f)

if ERROR_OCCURED:
    sys.exit("Error occurred")
else:
    sys.exit(0)


"""Tests for rebreak methods.

Specifically:
- ReflowSequence.rebreak()
"""

import logging

import pytest

from sqlfluff.core import Linter
from sqlfluff.utils.reflow.sequence import ReflowSequence

def parse_ansi_string(sql, config):
    """Parse an ansi sql string for testing."""
    linter = Linter(config=config)
    return linter.parse_string(sql).tree

@pytest.mark.parametrize(
    "raw_sql_in,raw_sql_out",
    [
        # Trivial Case
        ("select 1", "select 1"),
        # These rely on the default config being for leading operators
        ("select 1\n+2", "select 1\n+2"),
        ("select 1+\n2", "select 1\n+ 2"),  # NOTE: Implicit respace.
        ("select\n  1 +\n  2", "select\n  1\n  + 2"),
        ("select\n  1 +\n  -- comment\n  2", "select\n  1\n  -- comment\n  + 2"),
        # These rely on the default config being for trailing commas
        ("select a,b", "select a,b"),
        ("select a\n,b", "select a,\nb"),
        ("select\n  a\n  , b", "select\n  a,\n  b"),
        ("select\n    a\n    , b", "select\n    a,\n    b"),
        ("select\n  a\n    , b", "select\n  a,\n    b"),
        ("select\n  a\n  -- comment\n  , b", "select\n  a,\n  -- comment\n  b"),
    ],
)
def test_reflow__sequence_rebreak_root(raw_sql_in, raw_sql_out, default_config, caplog):
    """Test the ReflowSequence.rebreak() method directly.

    Focused around a whole segment.
    """
    root = parse_ansi_string(raw_sql_in, default_config)
    print(root.stringify())
    seq = ReflowSequence.from_root(root, config=default_config)
    for idx, elem in enumerate(seq.elements):
        print(idx, elem)

    with caplog.at_level(logging.DEBUG, logger="sqlfluff.rules.reflow"):
        new_seq = seq.rebreak()

    print(new_seq.get_fixes())
    assert new_seq.get_raw() == raw_sql_out

@pytest.mark.parametrize(
    "raw_sql_in,target_idx,seq_sql_in,seq_sql_out",
    [
        ("select 1+\n(2+3)", 4, "1+\n(", "1\n+ ("),
        ("select a,\n(b+c)", 4, "a,\n(", "a,\n("),
        ("select a\n  , (b+c)", 6, "a\n  , (", "a,\n  ("),
        # Here we don't have enough context to rebreak it so
        # it should be left unaltered.
        ("select a,\n(b+c)", 6, ",\n(b", ",\n(b"),
        # This intentionally targets an incomplete span.
        ("select a<=b", 4, "a<=", "a<="),
    ],
)
def test_reflow__sequence_rebreak_target(
    raw_sql_in, target_idx, seq_sql_in, seq_sql_out, default_config, caplog
):
    """Test the ReflowSequence.rebreak() method directly.

    Focused around a target segment. This intentionally
    stretches some of the span logic.
    """
    root = parse_ansi_string(raw_sql_in, default_config)
    print(root.stringify())
    target = root.raw_segments[target_idx]
    print("Target: ", target)
    seq = ReflowSequence.from_around_target(target, root, config=default_config)
    for idx, elem in enumerate(seq.elements):
        print(idx, elem)
    assert seq.get_raw() == seq_sql_in

    with caplog.at_level(logging.DEBUG, logger="sqlfluff.rules.reflow"):
        new_seq = seq.rebreak()

    print(new_seq.get_fixes())
    assert new_seq.get_raw() == seq_sql_out


""" Search page route. """

import time

from flask import render_template
from flask import current_app as app
from flask import request

from hubgrep.constants import PARAM_OFFSET, PARAM_PER_PAGE, FORM_ARGS
from hubgrep.lib.pagination import get_page_links
from hubgrep.lib.manticore import ManticoreSearch, SearchMeta
from hubgrep.lib.manticore import UserError
from hubgrep.lib.search_form import SearchForm
from hubgrep.frontend_blueprint import frontend

import logging

logger = logging.getLogger(__name__)

@frontend.route("/")
def search():
    total_time = time.time()
    results_offset = int(request.args.get(PARAM_OFFSET, 0))
    results_per_page = int(
        request.args.get(PARAM_PER_PAGE, app.config["PAGINATION_PER_PAGE_DEFAULT"])
    )
    form = SearchForm(
        search_phrase=request.args.get(FORM_ARGS.search_phrase, ""),
        exclude_service_checkboxes=SearchForm.get_request_service_checkboxes(),
        exclude_forks=request.args.get(FORM_ARGS.exclude_forks, "") == "on",
        exclude_archived=request.args.get(FORM_ARGS.exclude_archived, "") == "on",
        exclude_mirror=request.args.get(FORM_ARGS.exclude_mirror, "") == "on",
        exclude_empty=request.args.get(FORM_ARGS.exclude_empty, "") == "on",
        created_after=request.args.get(FORM_ARGS.created_after, ""),
        created_before=request.args.get(FORM_ARGS.created_before, ""),
        updated_after=request.args.get(FORM_ARGS.updated_after, ""),
        pushed_after=request.args.get(FORM_ARGS.pushed_after, ""),
    )
    results_page = []
    user_errors = []
    pagination_links = []
    meta = SearchMeta([])
    if form.search_phrase:
        try:
            results_page, meta = ManticoreSearch.search(
                form.search_phrase,
                results_offset=results_offset,
                results_per_page=results_per_page,
                exclude_hosting_service_ids=form.get_excluded_hosting_service_ids(),
                exclude_forks=form.exclude_forks,
                exclude_archived=form.exclude_archived,
                exclude_mirror=form.exclude_mirror,
                exclude_empty=form.exclude_empty,
                created_after=form.created_after_dt,
                created_before=form.created_before_dt,
                updated_after=form.updated_after_dt,
                pushed_after=form.pushed_after_dt,
            )
        except UserError as e:
            user_errors.append(e)

        if meta.warning:
            user_errors.append(meta.warning)

        pagination_links = get_page_links(request.full_path, results_offset, results_per_page, meta.total)

    template_path = (
        "search/search_list.html" if form.search_phrase else "search/landing_page.html"
    )
    # obviously we are missing the time it takes to render and respond
    # - but we measure our own "total" to give a more realistic time
    total_time = time.time() - total_time
    logger.info(f"search backend-processing took {total_time}s - engine-only took {meta.time}s")
    return render_template(
        template_path,
        form=form,
        search_results=results_page,
        search_meta=meta,
        total_time=total_time,
        pagination_links=pagination_links,
        user_errors=user_errors,
    )


import gzip
import json
import sys

import networkx as nx
import numpy as np
import osmnx as ox
import pandas as pd
import requests
import scipy as sp

import geomadi.geo_enrich as g_e
import geomadi.geo_octree as g_o

gO = g_o.h3tree()

def simplifyGraph(G, connection="strong"):
    """simplify a graph"""
    # G = ox.load_graphml(filename=fName)
    print("G nodes: " + str(len(G.nodes())))
    print("G edges: " + str(len(G.edges())))
    G_simp = ox.simplify_graph(G)
    print('simp_G:')
    print("G simp nodes: " + str(len(G_simp.nodes())))
    print("G simp edges: " + str(len(G_simp.edges())))
    if connection == "strong":
        G_connect = max(nx.weakly_connected_component_subgraphs(G_simp), key=len)
    elif connection == "weak":
        G_connect = max(nx.strongly_connected_component_subgraphs(G_simp), key=len)
    elif connection == "undirect":
        G_connect = max(nx.connected_component_subgraphs(G_simp.to_undirected()), key=len)
    else:
        G_connect = G_simp
    print('connected_G:')
    print("G connected nodes: " + str(len(G_connect.nodes())))
    print("G connected edges: " + str(len(G_connect.edges())))
    G_proj = ox.project_graph(G_connect, to_crs={'init': 'epsg:4326'})
    print('proj done')
    return G_proj

def removeType(G, move_type="driver"):
    """remove fclass types"""
    print("set move type %s" % (move_type))
    l = list(G.edges(data=True))
    tL, n = np.unique([x[2]['highway'] for x in l], return_counts=True)
    tL = ['cycleway', 'footway', 'living_street', 'motorway', 'motorway_link', 'path', 'pedestrian', 'primary',
          'residential', 'secondary', 'secondary_link', 'service', 'steps', 'tertiary', 'track', 'track_grade1',
          'track_grade3', 'trunk', 'trunk_link', 'unclassified']
    if move_type == "pedestrian":
        tL = ['cycleway', 'footway', 'living_street', 'path', 'pedestrian', 'residential', 'primary', 'secondary',
              'secondary_link', 'tertiary', 'trunk', 'trunk_link']
    if move_type == "driver":
        tL = ['living_street', 'motorway', 'motorway_link', 'primary', 'residential', 'secondary', 'secondary_link',
              'tertiary', 'trunk', 'trunk_link']
    G1 = G.copy()
    G1.graph['crs'] = {'init': 'epsg:4326'}
    G1.graph['name'] = move_type + " network"
    for i, j, x in G.edges(data=True):
        if x['highway'] in tL: continue
        G1.remove_edge(i, j)
    idL = []
    for i, j in G1.edges():
        idL.append(i)
        idL.append(j)
    idL = np.unique(idL)
    for i in G.nodes():
        if not i in idL:
            G1.remove_node(i)
    return G1

def composeGraph(graphL):
    """compose a list of graphs into a single one"""
    composed_G = nx.compose_all(graphL)
    return composed_G

def composeFiles(fileL):
    """compose a list of graphs from files"""
    graphL = []
    for f in fileL:
        graph1 = ox.load_graphml(filename=f)
    composed_G = nx.compose_all(graphL)
    simp_G = ox.simplify_graph(composed_G)
    return composed_G

def segment2graph(geo, isPlot=False):
    """create a graph from a collection of lines"""
    print("building the graph")
    gO = g_o.h3tree()
    G = nx.MultiDiGraph()
    # G = nx.Graph()
    G.graph['crs'] = {'init': 'epsg:4326'}
    G.graph['name'] = "local network"
    for i, g in geo.iterrows():
        seg = list(g['geometry'].coords)
        for start, end, j in zip(seg[:-1], seg[1:], range(len(seg))):
            # line = sh.geometry.LineString([start,end])
            dist = g_e.haversine(start[0], start[1], end[0], end[1])
            # j1, j2 = g['osm_id']+"_"+str(j), g['osm_id']+"_"+str(j+1)
            j1, j2 = gO.encode(start[0], start[1], precision=13), gO.encode(end[0], end[1], precision=13)
            G.add_edge(j1, j2, key=0, src=j1, trg=j2, speed=g['maxspeed'], highway=g['fclass'], name=g['name'],
                       length=dist)  # ,geometry=line,osm_id=g['osm_id'])
            G.add_node(j1, pos=start, x=start[0], y=start[1], key=0)  # ,osmid=j1)
            G.add_node(j2, pos=end, x=end[0], y=end[1], key=0)  # ,osmid=j2)
    if isPlot:
        ox.plot_graph(G)
    return G

def line2graph(geo, isPlot=False):
    """create a graph from a collection of lines"""
    gO = g_o.h3tree()
    G = nx.MultiDiGraph()
    # G = nx.Graph()
    G.graph['crs'] = {'init': 'epsg:4326'}
    G.graph['name'] = "local network"
    for i, g in geo.iterrows():
        seg = list(g['geometry'].coords)
        start, end = seg[0], seg[-1]
        dist = g_e.haversine(start[0], start[1], end[0], end[1])
        j1, j2 = gO.encode(start[0], start[1], precision=13), gO.encode(end[0], end[1], precision=13)
        G.add_edge(j1, j2, key=0, speed=g['maxspeed'], highway=g['fclass'], name=g['name'], length=dist)
        G.add_node(j1, pos=start, x=start[0], y=start[1], key=0)
        G.add_node(j2, pos=end, x=end[0], y=end[1], key=0)
    if isPlot:
        ox.plot_graph(G)
    print("built the graph size %d" % (G.size()))
    return G

def downloadGraph(city="Salò, Italy"):
    """download graph from city"""
    G = ox.graph_from_place(city)
    G = ox.project_graph(G)
    ox.plot_graph(G)
    return G

def lineLenght(l):
    """length of a line in kilometers"""
    l1 = [[x, y] for x, y in zip(l.xy[0], l.xy[1])]
    length = sum([g_e.haversine(x[0], x[1], y[0], y[1]) for x, y in zip(l1[:-1], l1[1:])])
    return length

def getNearestNode(G, coord_x, coord_y):
    """get nearest node, if actual node is not in the graph"""
    coords = (coord_x, coord_y)
    node = ox.get_nearest_node(G, coords, method='euclidean')
    return node

def getNeighbors(G, posL):
    """get nearest node, if actual node is not in the graph"""
    pos = [(x['x'], x['y']) for i, x in G.nodes(data=True)]
    tree = sp.spatial.KDTree(pos)
    nearest_dist, nearest_ind = tree.query(posL)
    posI = {}
    for i, j in enumerate(G.nodes()): posI[i] = j
    posI = [posI[x] for x in nearest_ind]
    neiL = [G.node[x] for x in posI]
    return posI

def weightGraph(graph):
    keys = {}
    for edge in (graph.edges(data=True)):
        attrs = {}
        if edge[2]['highway'] == 'motorway':
            attrs['weight'] = edge[2]['length'] * 1
        elif edge[2]['highway'] == 'primary':
            attrs['weight'] = edge[2]['length'] * 1.5
        elif edge[2]['highway'] == 'secondary':
            attrs['weight'] = edge[2]['length'] * 1.8
        else:
            attrs['weight'] = edge[2]['length'] * 3
        keyy = (edge[0], edge[1], 0)
        keys[keyy] = attrs
    nx.set_edge_attributes(graph, keys)
    return graph

def weightSpeed(G):
    """weight the graph according to street speed"""
    wL = {'motorway': 3, 'motorway_link': 3, 'primary': 2, 'residential': 0.5, 'secondary': 2, 'secondary_link': 2,
          'tertiary': 1.5, 'trunk': 2, 'trunk_link': 2}
    keys = {}
    for u, v, k, edge in (G.edges(keys=True, data=True)):
        s = edge['speed']
        l = edge['length']
        t = edge['highway']
        if isinstance(s, list): s = s[0]
        if isinstance(l, list): l = l[0]
        if isinstance(t, list): t = t[0]
        t = wL[t]
        s = max(s, 30)  # kmh
        l = max(l, 0.1)  # km
        w = s / l * 0.01 * t
        G[u][v][k].update(weight=w)
    return G

def calcPair(G, route):
    """pair distance between points"""
    if len(route) < 2: return False, [0, 0, 0, 0]
    length, weight, duration = 0, 0, 0
    for i, j in zip(route[:-1], route[1:]):
        edgeData = G.get_edge_data(i, j)[0]
        length = length + edgeData['length']
        weight = weight + edgeData['weight']
        if isinstance(edgeData['speed'], list):
            time = edgeData['length'] / max(min(edgeData['speed']), 1)  # *(1.-edgeData['weight']))
        else:
            time = edgeData['length'] / max(edgeData['speed'], 1)  # *(1.-edgeData['weight']))
        duration = duration + time
    weight = weight / (len(route) - 1)
    # if length > 3: return False, [0,0,0,0] #long distances are useless
    return True, [route[0], route[-1], length, weight, duration]

def calcRoute(G, start, destL, colL):
    node = start['node']
    try:  # calculate the route from the OSM node to all nodes in the network
        route_dij = nx.single_source_dijkstra_path(G=G, source=node, weight='weight')
    except:
        print("Start node " + str(node) + " not found")
        nearest_node = getNearestNode(node, start['y'], start['x'])
        print("Using nearest node instead, nodeid: " + str(nearest_node))
        route_dij = nx.single_source_dijkstra_path(G=G, source=nearest_node, weight='weight')

    routeL = [x for i, x in route_dij.items() if i in destL]
    pairL = []
    for route in routeL:
        status, link = calcPair(G, route)
        if status: pairL.append(link)

    pairL = pd.DataFrame(pairL, columns=colL)
    return pairL, len(routeL) / len(destL)

def routeOsrm(start, end):
    """routing-with-local-osrm"""
    # 'docker run -t -i -p 5000:5000 -v "${PWD}:/data" osrm/osrm-backend osrm-routed --algorithm mld /data/berlin-latest.osrm')
    x1, y1 = gO.decode(start)
    x2, y2 = gO.decode(end)
    fromP = "%f,%f" % (x1, y1)
    toP = "%f,%f" % (x2, y2)
    resq = requests.get("http://127.0.0.1:5000/route/v1/driving/" + fromP + ";" + toP + "?steps=true")
    route = resq.json()
    route = route['routes'][0]
    return [start, end, route['distance'], route['weight'], route['duration']]

def progress(count, total, status='', delta_time=0, completion=1.):
    """progressbar"""
    bar_len = 60
    filled_len = int(round(bar_len * count / float(total)))
    percents = round(100.0 * count / float(total), 1)
    bar = '=' * filled_len + '-' * (bar_len - filled_len)
    sys.stdout.write('[%s] %s%% %s %.2fh comp %.2f\r' % (bar, percents, status, delta_time, completion))
    sys.stdout.flush()

def save(G, fName):
    """suggest different file formats"""
    nx.write_gml(G, fName + ".gml")
    # G = nx.read_gml(fName+".gml")
    if False:
        ox.save_graphml(G, filename=fName, folder=baseDir + "gis/graph/")
        ox.save_gdf_shapefile(graph, filename=fName + ".gdf", folder=baseDir + "gis/graph/")
        ox.save_graphml(G, filename=gName, folder=baseDir + "gis/graph/")
        G = ox.load_graphml(filename=gName, folder=baseDir + "gis/graph/")
        ox.save_gdf_shapefile(G, filename=fName + ".gdf", folder=baseDir + "gis/graph/")
        G = ox.load_graphml(filename=fName + ".gdf", folder=baseDir + "gis/graph/")
        nx.readwrite.write_shp(G, baseDir + "gis/graph/berlin_street")
        nx.write_graphml_lxml(G, baseDir + "fName" + ".graphml")

def saveJson(G, fName, indent=0):
    """save a simple json"""
    l = [(u, v, k, x) for u, v, k, x in G.edges(keys=True, data="geometry") if x != None]
    print("geometry elements %d" % len(l))
    for i in l:
        del G[i[0]][i[1]][i[2]]['geometry']
    data = dict(nodes=list(G.nodes(data=True)), edges=list(G.edges(data=True)))
    json_str = json.dumps(data, indent=indent) + "\n"
    json_bytes = json_str.encode('utf-8')
    with gzip.GzipFile(fName, 'w') as fout:
        fout.write(json_bytes)

def loadJson(fName):
    """load a graph from a json"""
    G = nx.MultiGraph()
    G.graph['crs'] = {'init': 'epsg:4326'}
    G.graph['name'] = "local network"
    with gzip.GzipFile(fName, 'r') as fin:
        json_bytes = fin.read()
    json_str = json_bytes.decode('utf-8')
    data = json.loads(json_str)
    G.add_nodes_from(data['nodes'])
    G.add_edges_from(data['edges'])
    return G


# Dependência necessária
# pip install pefile

import os , sys
import pefile

# Escreva um script em Python  que receba como entrada um arquivo ou diretório
# e enumere a seções executáveis do(s) binário(s), imprimindo na saída padrão um dicionário de listas,
# onde a chave é o nome do binário e o valor é uma lista de seções executáveis.

# Pequena consistência para verificar se são passados pelo menos 2 argumentos
#if (len(sys.argv) != 3):
#    print('Devem ser passados 2 argumentos, os dois .exe a serem comparados !')
#    exit()

executavel1 = str(sys.argv[1])
executavel2 = str(sys.argv[2])

#executavel1 = '/home/pulgatti/Dropbox/Doutorado/Materias/CI1030-ERE2-CienciaDeDados/ping.exe'
#executavel2 = '/home/pulgatti/Dropbox/Doutorado/Materias/CI1030-ERE2-CienciaDeDados/notepad++.exe'
nomeexec1 = executavel1[executavel1.rfind('/')+1:]
nomeexec2 = executavel2[executavel2.rfind('/')+1:]

dicionario = {}

def carrega_secao(arquivo):
        pe =  pefile.PE(arquivo)
        lista_secao = ''
        for section in pe.sections:
            lista_secao = lista_secao + str(section.Name)
        # Limpa da string caracteres não representativos retornados pela ferramenta
        lista_secao = lista_secao.replace("'b'", ',')
        lista_secao = lista_secao.replace("b'", '')
        lista_secao = lista_secao.replace("\\x00'", '')
        lista_secao = lista_secao.replace("\\x00", '')
        return lista_secao

# Carrega um dicionário com a lista das seções de cada um dos .exe recebidos como parâmetro
# Sim, eu sei que está feio codificar no braço as variáveis...
dicionario[nomeexec1] = [str(carrega_secao(executavel1))]
dicionario[nomeexec2] = [str(carrega_secao(executavel2))]

# transforma o valor das entrada que está em uma string para uma lista
for item in dicionario:
    dicionario[item] = str(dicionario[item][0]).split(",")

# imprime na saída padrão, os valores únicos e os comuns aos dois executáveis
print('=================')
print('Seções únicas por .exe')
print('=================')

print( nomeexec1, ' : ', set(dicionario[nomeexec1]) - set(dicionario[nomeexec2]) )
print( nomeexec2, ' : ', set(dicionario[nomeexec2]) - set(dicionario[nomeexec1]) )

print('=================')
print('Seções comuns a todos os .exe analizados')
print('=================')

print(  set(dicionario[nomeexec1]) & set(dicionario[nomeexec2]) )


# !/usr/bin/env python
# -*- coding: utf-8 -*-
"""
MathGenerator class of the MathEngine module.
"""

__authors__ = ["Luana Juhl", "Lukas Schult"]
__contact__ = "it16156@lehre.dhbw-stuttgart.de"
__credits__ = ["Luana Juhl", "Lukas Schult"]
__date__ = "2021/04/27"
__deprecated__ = False
__email__ = "it16156@lehre.dhbw-stuttgart.de"
__maintainer__ = "developer"
__status__ = "Released"
__version__ = "1.0"

import numpy as np
import operator
import json

class MathGenerator:
    """
    A Generator of mathematical terms and their textual representaion.
    Also provides a difficulty selection to produce various terms with a selection of answers to the term.
    """

    def __init__(self):
        self.rng = np.random.default_rng()

        self.lvl_generator_functions = {0: self.generate_add_sub_lvl,
                                        1: self.generate_add_sub_mult_lvl,
                                        2: self.generate_random_add_sub_lvl,
                                        3: self.generate_random_mult_div_lvl}
        self.range_modifiers = {0: 20,
                                1: 50,
                                2: 100}

    ops = {
        "+": operator.add,
        "-": operator.sub,
        "*": operator.mul,
        "/": operator.truediv
    }
    operations = ["+", "-", "×", "÷"]

    def get_term(self, difficulty):
        """
        Generates a term for a given difficulty level.
        :param difficulty: int Term level
        :return: serialized data of a term in a dictionary
        """
        if difficulty == 0:
            return self.generate_add_sub_lvl(possible_numbers=np.arange(start=0, stop=21, step=1), only_add=True)
        else:
            generator_selection, start, stop = self.determine_level_range(difficulty)
            if generator_selection == 1:  # add_sub_mul needs 2 ranges given in start and stop lists
                return self.lvl_generator_functions[generator_selection](
                    np.arange(start=start[0], stop=stop[0], step=1),
                    np.arange(start=start[1], stop=stop[1], step=1))
            else:
                return self.lvl_generator_functions[generator_selection](
                    np.arange(start=start[0], stop=stop[0], step=1))

    def determine_level_range(self, difficulty):
        """
        Determines which function to use to generate a term and with which number range it should be generated.
        :param difficulty: int Term level
        :return: generator_selection: level generating function, start + stop: lists for start and stop for np.arrange
        """
        modifier_selection = difficulty % 3
        generator_selection = int(difficulty / 3) % 4

        start = [0 - self.range_modifiers[modifier_selection]]
        stop = [self.range_modifiers[modifier_selection]]

        if generator_selection == 1:
            start.append(max(int(start / 5), 10) if start[0] > 0 else min(int(start[0] / 5), -10))
            stop.append(max(int(stop[0] / 5), 10) + 1)
        return generator_selection, start, stop

    def build_term_string(self, term_steps, operator_symbols):
        """
        Builds a String for a term with the given steps.
        :param term_steps: Steps to be inserted.
        :param operator_symbol: operator symbol to be inserted.
        :return: Term String.
        """
        term_string = f"{term_steps[0]}"
        for i, value in enumerate(term_steps[1:]):
            if type(operator_symbols) == list:
                operation = operator_symbols[i]
            else:
                operation = operator_symbols
            if value < 0 and operation == "-":  # auto convert  - - to +
                operation = "+"
                value = -value
            elif value < 0:  # and operation in ["+", "*", "/"]:  # use parentheses for negative values in mult and div
                value = f"({value})"
            term_string += f" {operation} {value}"
        return term_string + " = ?"

    def generate_add_sub_lvl(self, possible_numbers, only_add=False):
        """
        Generates a addition or subtraction term response.
        :param only_add: removes the random choice for the operator
        :param possible_numbers: All numbers that can occur.
        :return: dictionary of term string, answer options and solution_index
        """
        number_count = 2
        if only_add:
            operator_symbol = "+"
        else:
            operator_symbol = self.rng.choice(["+", "-"])
        term = self.generate_add_or_sub(number_count, possible_numbers, operator_symbol)
        answers, solution_index = self.generate_answers(term[operator_symbol][0], possible_numbers)
        return self.create_response_dict(self.build_term_string(term[operator_symbol][1], operator_symbol),
                                         answers,
                                         solution_index
                                         )

    def generate_add_sub_mult_lvl(self, possible_numbers_add_sub, possible_numbers_mult):
        """
            Generates a addition, subtraction or multiplication term response.
            :param possible_numbers_add_sub: All numbers that can occur for addition and subtraction terms.
            :param possible_numbers_mult: All numbers that can occur for multiplication terms.
            :return: dictionary of term string, answer options and solution_index
        """
        operator_symbol = self.rng.choice(["+", "-", "*"])
        if operator_symbol in ["+", "-"]:
            number_count = self.rng.choice([2, 3])
            term = self.generate_add_or_sub(number_count, possible_numbers_add_sub, operator_symbol)
            answers, solution_index = self.generate_answers(term[operator_symbol][0], possible_numbers_add_sub)

        else:
            term = self.generate_multiplication_value_pair(possible_numbers_mult)
            answers, solution_index = self.generate_answers(term[operator_symbol][0], possible_numbers_mult)
        return self.create_response_dict(self.build_term_string(term[operator_symbol][1], operator_symbol),
                                         answers,
                                         solution_index
                                         )

    def generate_random_add_sub_lvl(self, possible_numbers):
        """
            Generates a mixed term response (combination between addition and subtraction of up to 4 numbers)
            :param possible_numbers: All numbers that can occur.
            :return: dictionary of term string, answer options and solution_index
        """

        term = self.generate_mixed_add_sub(self.rng.choice([3, 4]), possible_numbers)
        answers, solution_index = self.generate_answers(term[0], possible_numbers)
        return self.create_response_dict(self.build_term_string(term[1], operator_symbols=term[2]),
                                         answers,
                                         solution_index
                                         )

    def generate_random_mult_div_lvl(self, possible_numbers):
        """
            Generates a multiplication or division term response.
            :param possible_numbers: All numbers that can occur.
            :return: dictionary of term string, answer options and solution_index
        """
        term_type = self.rng.choice(["*", "/"])
        if term_type == "*":
            term = self.generate_multiplication_value_pair(possible_numbers)
        else:
            term = self.generate_division_value_pair(possible_numbers)
        answers, solution_index = self.generate_answers(term[term_type][0], possible_numbers)
        return self.create_response_dict(self.build_term_string(term[term_type][1], operator_symbols=term_type),
                                         answers,
                                         solution_index
                                         )

    def create_response_dict(self, term_string, answers, solution_index):
        """
            Generates a multiplication or division term response.
            :param term_string: string version of the term (for display)
            :param answers: a json compatible list of possible answers
            :param solution_index: the list index for the correct solution
            :return: dictionary of term string, answer options and solution_index
        """
        return {
            "term": term_string,
            "answers": json.dumps(answers.tolist()),
            "solution_index": solution_index
        }

    def generate_add_or_sub(self, step_count, num_range, operator_symbol, start=0):
        """
        Generates a term for addition or subtraction with a variable number of values that lead to the result.
        :param step_count: Number of values that will lead to the term result.
        :param num_range: The number range for which the term will be generated in.
        :param operator_symbol: + or - operator symbol
        :param start: defines the first value of the term if not 0
        :return: dict with { operation_symbol: [result, array of steps]}
        """
        operation_func = self.ops[operator_symbol]
        steps = np.zeros(step_count, dtype=int)
        steps[0] = start
        while steps[0] == 0:
            choice = self.rng.choice(num_range)
            if operation_func(choice, step_count) in num_range:
                steps[0] = choice

        def get_result():
            step_sum = steps[0]
            for step in steps[1:]:
                step_sum = operation_func(step_sum, step)
            return step_sum

        for i in range(step_count - 1):
            while steps[i + 1] == 0:
                choice = self.rng.choice(num_range)
                if operation_func(operation_func(get_result(), choice), (step_count - (i + 2))) in num_range:
                    steps[i + 1] = choice

        return {operator_symbol: [get_result(), steps]}

    def is_prime(self, n):
        """
        Checks if given number is prime.
        :param n: number to check
        :return: True if prime else False
        """
        for i in range(2, n):
            if n % i == 0:
                return False
        return True

    def generate_multiplication_value_pair(self, num_range):
        """
        Generates a multiplication term with 2 values.
        :param num_range: The number range for which the term will be generated in.
        :return: dict with { operation_symbol: [result, array of steps]}
        """
        retry = True

        while retry:
            result = 0
            negative = False

            while result == 0:
                retry = False
                result = self.rng.choice(num_range)
                # check if result is prime and retry with a 90% chance

                if self.is_prime(abs(result)) and self.rng.choice([True, False], p=[0.9, 0.1]):
                    retry = True

            if result < 0:
                negative = True
                result = -result
            steps = np.zeros(2, dtype=int)

            divisors = []
            i = 1
            while i <= result:  # look for all divisors without residue
                if result % i == 0:
                    divisors.append(i)  # fill divisors list
                i += 1
            steps[0] = self.rng.choice(divisors)  # choose divisor at random for first operand
            steps[1] = result / steps[0]  # calculate the second operand
            if negative:
                negate_idx = self.rng.choice([0, 1])
                steps[negate_idx] = -steps[negate_idx]
                result = -result

            for step in steps:  # avoid terms with multiplication by 1 with 90% probability
                if step == 1 and self.rng.choice([True, False], p=[0.9, 0.1]):
                    retry = True
        return {"*": [result, steps]}

    def generate_division_value_pair(self, num_range):
        """
        Generates a division term with one numerator and one divisor.
        :param num_range: The number range for which the term will be generated in.
        :return: dict with { operation_symbol: [result, array of steps]}
        """
        numerator = 0
        divisor = 0
        while numerator == 0:  # ensures numerator will not be 0
            numerator = self.rng.choice(num_range)
            divisor = self.rng.choice(num_range)
            if divisor == 0:  # ensures not to divide by 0
                numerator = 0
            elif not (numerator / divisor).is_integer():  # only allow for integer values as result
                numerator = 0

        return {"/": [int(numerator / divisor), np.array([numerator, divisor])]}

    def generate_random_add_or_sub_pair(self, num_range):
        """
        Generates a random term with 2 steps and either + or - operation.
        :param num_range: The number range for which the term will be generated in.
        :return: dict with { operation_symbol: [result, array of steps]}
        """
        operation = self.rng.choice(["+", "-"])
        return self.generate_add_or_sub(2, num_range, operation)

    def generate_mixed_add_sub(self, step_count, num_range):
        """
        Generates a term with mixed operations. This includes at least one + and - .
        :param step_count: The number of steps for the requested term.
        :param num_range: The number range for which the term will be generated in.
        :return: Result of the term, steps of the term, operations between steps.
        """
        retry = True
        if step_count == 2:
            return self.generate_random_add_or_sub_pair(num_range)
        while retry:
            steps = np.zeros(step_count, dtype=dict)
            term = self.generate_random_add_or_sub_pair(num_range)  # generate the first term
            operations = list(term.keys())  # list to keep track of the operations between steps
            steps[0] = term[operations[0]][1][0]  # setting saving first and second step
            steps[1] = term[operations[0]][1][1]
            step = 2
            while 0 in steps:  # continues to add values from new terms to the steps
                term = self.generate_random_add_or_sub_pair(num_range)
                operations += list(term.keys())
                # checks if the first value of the term uses the last value of the steps
                if term[operations[step - 1]][1][0] == steps[step - 1]:
                    steps[step] = term[operations[step - 1]][1][1]  # adds the next value since the fist is in steps
                    step += 1
                else:
                    operations.pop()  # remove the operation from the list if the term does not fit the followup value
                if "-" in operations and "+" in operations:  # ensures that the term is mixed
                    retry = False
            result = steps[0]  # setting the initial value for the result calculation
            for i, operation in enumerate(operations, start=1):
                operation_func = self.ops[operation]
                result = operation_func(result, steps[i])  # dynamically use operations to calculate the result
            if not (result in num_range):  # ensures that the result is inside the given number range
                retry = True

        return result, steps, operations

    def generate_answers(self, solution, num_range):
        """
        Generates wrong answers for answer options.
        :param solution: the correct term result
        :param num_range: the boundaries for generated answers
        :return: tuple of answer list and correct answer index
        """
        answers = np.zeros(4)
        indexes = list(range(4))
        np.random.shuffle(indexes)
        answers[indexes[0]] = solution
        for i in [1, 2]:
            answer = solution + self.rng.choice(range(-5, 5))
            while answer == solution or answer not in num_range:
                answer = solution + self.rng.choice(range(-5, 5))
            answers[indexes[i]] = int(answer)
        answer = self.rng.choice(num_range)
        while answer == solution or answer not in num_range:
            answer = self.rng.choice(num_range)
        answers[indexes[3]] = int(answer)
        return answers, indexes[0]