import os
import json
import numpy as np
import argparse

pp = argparse.ArgumentParser(description='Format PASCAL 2012 metadata.')
pp.add_argument('--load-path', type=str, default='/home/julioarroyo/research_Eli_and_Julio/single-positive-multi-label-julio/data/pascal/', help='Path to a directory containing a copy of the PASCAL dataset.')
pp.add_argument('--save-path', type=str, default='/home/julioarroyo/research_Eli_and_Julio/single-positive-multi-label-julio/data/pascal/', help='Path to output directory.')
args = pp.parse_args()

catName_to_catID = {
    'person': 0,
    'bird': 1,
    'cat': 2,
    'cow': 3,
    'dog': 4,
    'horse': 5,
    'sheep': 6,
    'aeroplane': 7,
    'bicycle': 8,
    'boat': 9,
    'bus': 10,
    'car': 11,
    'motorbike': 12,
    'train': 13,
    'bottle': 14,
    'chair': 15,
    'diningtable': 16,
    'pottedplant': 17,
    'sofa': 18,
    'tvmonitor': 19
}

catID_to_catName = {catName_to_catID[k]: k for k in catName_to_catID}

ann_dict = {}
image_list = {'train': [], 'val': []}

for phase in ['train', 'val']:
    for cat in catName_to_catID:
        # this file tells you in what images does cat occurr
        with open(os.path.join(args.load_path, 'VOCdevkit', 'VOC2012', 'ImageSets', 'Main', cat + '_' + phase + '.txt'), 'r') as f:
            for line in f:
                cur_line = line.rstrip().split(' ')
                image_id = cur_line[0]
                label = cur_line[-1]
                image_fname = image_id + '.jpg'
                # if cat occurs, save it in ann_dict, which maps from image to catID that occur in the image
                # image_list holds all images with annotated objects in them, classified by train or val
                if int(label) == 1:
                    if image_fname not in ann_dict:
                        ann_dict[image_fname] = []
                        image_list[phase].append(image_fname)
                    ann_dict[image_fname].append(catName_to_catID[cat])
    # create label matrix:
    image_list[phase].sort()
    num_images = len(image_list[phase])
    label_matrix = np.zeros((num_images, len(catName_to_catID)))
    for i in range(num_images):
        cur_image = image_list[phase][i]
        label_indices = np.array(ann_dict[cur_image])
        label_matrix[i, label_indices] = 1.0 # puts a 1.0 at (i, j) where i is the i-th image and j is the index of the category
    np.save(os.path.join(args.save_path, 'formatted_' + phase + '_labels.npy'), label_matrix)
    np.save(os.path.join(args.save_path, 'formatted_' + phase + '_images.npy'), np.array(image_list[phase]))


"""
    Collect and prepare the CMC data for dataset
    (similar to create_tiny_datastet.py but only for CMC)
"""

path_manual_annotations = r"./annotations/shot-annotations_manual"
path_camera_annotations = r"./eval_cmc/all_annotations_per_vid_09032022"
path_cmc_output = r"./annotations/camera_annotations_manual"

import os
import json
import glob
import csv

def main():
    if not os.path.isdir(path_cmc_output):
        os.mkdir(path_cmc_output)

    annotations = glob.glob(os.path.join(path_manual_annotations, "*-shot_annotations.json"))
    for path in annotations:
        vid = os.path.split(path)[-1].split('-')[0]

        with open(path) as f:
            manual_annotations = json.load(f)

        path_cam_ann = os.path.join(path_camera_annotations, f"{vid}.csv")

        # Only continue if a camera movement exists
        if not os.path.isfile(path_cam_ann):
            continue

        with open(path_cam_ann) as f:
            csv_reader = csv.DictReader(f, delimiter=';')
            all_camera_annotations = [dict(row) for row in csv_reader]

        # Collect camera annotations
        camera_annotations = []
        for annotation in all_camera_annotations:
            corresponding_shot = list(filter(lambda shot: shot["inPoint"] <= int(annotation["start"]) - 1 and shot["outPoint"] >= int(annotation["stop"]) - 1, manual_annotations))
            # print(annotation)
            # print(corresponding_shot)
            if not len(corresponding_shot) == 1:
                print("VID: ", vid, "Start:", annotation["start"], "Stop:", annotation["stop"])
                continue

            camera_annotations.append({
            "shotId": corresponding_shot[0]["shotId"],
            # Note that SID in the CMCs is actually cmId (see below)
            "cmId": int(annotation["sid"]),
            "Start": int(annotation["start"]) - 1,
            "Stop": int(annotation["stop"]) - 1,
            "class_name": annotation["class_name"]})

        with open(os.path.join(path_cmc_output, f"{vid}-sequence_annotations.json"), 'w') as outfile:
            json.dump(camera_annotations, outfile, indent=4)

if __name__ == "__main__":
    main()


import collections
import multiprocessing

def calculate_price_in_rupees(DishName):
    return {
        'dish': DishName.dish,
        'price': round(DishName.price * 77.85, 2)
    }

DishName = collections.namedtuple('DishName', [
    'dish',
    'price',
])
if __name__ == "__main__":
    DishNames = (
        DishName(dish='Shrimp Carbonara', price=1815),
        DishName(dish='Lobster Bisque', price=1882),
        DishName(dish='Spaghetti with Meatballs', price=1867),
        DishName(dish='Tom Kha Ghai', price=1930),
        DishName(dish='Cordon Bleu', price=1939),
        DishName(dish='Coq au Vin', price=1928),
        DishName(dish='Pomme Frites', price=1951),
    )

pool = multiprocessing.Pool()
result = pool.map(calculate_price_in_rupees, DishNames)
print(result)
'''
[{'dish': 'Shrimp Carbonara', 'price': 141297.75},
{'dish': 'Lobster Bisque', 'price': 146513.7},
{'dish': 'Spaghetti with Meatballs', 'price': 145345.95},
{'dish': 'Tom Kha Ghai', 'price': 150250.5},
{'dish': 'Cordon Bleu', 'price': 150951.15},
{'dish': 'Coq au Vin', 'price': 150094.8},
{'dish': 'Pomme Frites', 'price': 151885.35}]
'''


import torch
import numpy as np

def createDictLabels(labels):

    """
    Creates dictionaries that fits data with non-sequential labels into a
    sequential order label from [0...nClasses].
    :param labels: all the non-sequential labels
    :return: dict that converts from non-sequential to sequential,
             dict that converts from sequential to non-sequential
    """

    # Re-arange the Target vectors between [0..nClasses_train]
    labels = labels.numpy()
    unique_labels = np.unique(labels)
    dictLabels = {val: i for i, val in enumerate(unique_labels)}
    dictLabelsInverse = {i: val for i, val in enumerate(unique_labels)}
    return dictLabels,dictLabelsInverse

def fitLabelsToRange(dictLabels,labels):

    """
    Converts Tensor values to the values contained in the dictionary
    :param dictLabels: dictionary with the conversion values
    :param labels: Tensor to convert
    :return: Tensor with the converted labels.
    """
    labels = labels.numpy()
    unique_labels = np.unique(labels)
    labels_temp = np.array(labels)
    for i in dictLabels.keys():
        labels_temp[labels == i] = dictLabels[i]
    labels = labels_temp
    return torch.from_numpy(labels)

def unflattenParams(model,flatParams):
    flatParams = flatParams.squeeze()
    indx = 0
    for param in model.net.parameters():
        lengthParam = param.view(-1).size()[0]
        param.data = flatParams[indx:indx+lengthParam].view_as(param).data
        indx = indx + lengthParam
    a = 0


# tutorial based on this YT video: https://www.youtube.com/watch?v=lLPdRRy7dfE&list=PL3JVwFmb_BnRKqcbtl2hHL5GIQOHX-sC5&index=1
import os
from google.cloud import bigquery

os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = 'CloudDemo_AS_Service.json'

client = bigquery.Client()

sql_query = """
SELECT station_id, name, dockcount
FROM `bigquery-public-data.san_francisco.bikeshare_stations`
LIMIT 50
"""

query_job = client.query(sql_query)

for row in query_job.result():
    print(row)

print(query_job.to_dataframe())


#%%
from collections import defaultdict
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from scipy import stats
import seaborn as sns

#%%
models = ['gpt2',
          'gpt2-medium',
          'gpt2-large',
          'gpt2-xl',
          'opt-2.7b',
          'opt-6.7b',
          'opt-13b',
          'llama-7B',
          'llama-13B',
          'llama-30B',
          ]

for model in models:
    df = pd.read_csv(f'experiment3/{model}.csv')
    animacy_norms = pd.read_csv('experiment3/animacy_norms.csv')
    verb_df = df[df.prompt_type == 'VERB']
    df['kl_diff'] = df['animate_no_context_inanimate_no_context_kl'] - df['animate_no_context_experimental_kl']
    df['emd_diff'] = df['animate_no_context_inanimate_no_context_emd'] - df['animate_no_context_experimental_emd']

    ###
    mental_animacy = defaultdict(int)
    mental_animacy.update({word:score for word, score in zip(animacy_norms['Word'], animacy_norms['AnimMental'])})
    physical_animacy = defaultdict(int)
    physical_animacy.update({word:score for word, score in zip(animacy_norms['Word'], animacy_norms['AnimPhysical'])})

    ###
    nouns_list = list(set(df['noun']))

    noun_summary_dict = {'noun': nouns_list}
    for column in df.columns:
        if df[column].dtype == np.float64:
            noun_summary_dict[column] = [df[(df.noun == noun) & (~pd.isna(df[column]))][column].mean() for noun in nouns_list]
    noun_summary_df = pd.DataFrame.from_dict(noun_summary_dict)
    noun_summary_df['AnimMental'] = [mental_animacy[noun] if noun in mental_animacy else np.nan for noun in nouns_list]
    noun_summary_df['AnimPhysical'] = [physical_animacy[noun] if noun in mental_animacy else np.nan for noun in nouns_list]

    verbs_list = list(set(df['verb']))
    verbs_list = list(set(df['verb']))
    verb_summary_dict = {'verb': verbs_list}
    for column in df.columns:
        if df[column].dtype == np.float64:
            verb_summary_dict[column] = [df[(df.verb == verb) & (~pd.isna(df[column]))][column].mean() for verb in verbs_list]
    verb_summary_df = pd.DataFrame.from_dict(verb_summary_dict)

    ###
    # LOADING VERBS

    verb_info = pd.read_csv('verbs.csv')

    verb_types = {}
    verb_frequencies = {}

    for verb, verb_frequency, verb_type in zip(verb_info['verb'], verb_info['frequency'], verb_info['type']):
        verb_types[verb] = verb_type
        verb_frequencies[verb] = verb_frequency

    verb_summary_df['verb_frequency'] = [verb_frequencies[verb] for verb in verb_summary_df['verb']]
    verb_summary_df['verb_frequency_color'] = ['r' if freq == 'HIGH' else ('y' if freq == 'HIGH-MID' else 'b') for freq in verb_summary_df['verb_frequency']]
    verb_summary_df['verb_category'] = [verb_types[verb] for verb in verb_summary_df['verb']]
    verb_summary_df['verb_category_color'] = ['r' if cat == 'physical' else 'b' for cat in verb_summary_df['verb_category']]

    ###
    plot_df = df.copy()
    plot_df['Animacy Divergence'] = plot_df['animate_no_context_experimental_kl'] * 10000
    fig, axs_dict = plt.subplot_mosaic('ABC', sharey=True) #plt.subplots(1, 3, sharey=True)
    axs = list(axs_dict.values())
    (ax1, ax2, ax3) = axs
    sns.kdeplot(data=plot_df, x="Animacy Divergence", hue='prompt_type', common_norm=False, ax=ax1, hue_order=['VERB', 'ADJ'])
    sns.kdeplot(data=plot_df, x="Animacy Divergence", hue='verb_category', common_norm=False, ax=ax2, hue_order=['physical', 'psychological'])
    sns.kdeplot(data=plot_df, x="Animacy Divergence", hue='verb_frequency', common_norm=False, ax=ax3, hue_order=['HIGH', 'HIGH-MID', 'MID'])
    for k, ax in axs_dict.items():
        ax.ticklabel_format(axis='x', style='sci', scilimits=(-4, 6))
        ax.get_xaxis().get_label().set_visible(False)
        ax.text(0.05, 0.85, k, transform=ax.transAxes,
            size=20, weight='bold')

    fig.supxlabel("Animacy Divergence (1e-4)")
    fig.set_size_inches(8.5, 3)
    fig.suptitle("Animacy Divergence Distribution by Prompt Type, Verb Category, and Verb-Human Co-occurrence")
    fig.tight_layout()
    fig.show()
    if model == 'llama-7B':
        fig.savefig(f'paper-plots/distributions-{model}.pdf')
    fig.suptitle(f"Animacy Divergence Dist. by Prompt Type, Verb Category, and Verb-Human Co-occ. ({model})")
    #fig.savefig(f'paper-plots-png/distributions-{model}.png')
    fig.savefig(f'exp3-paper-plots/distributions-{model}.pdf')

    ###
    col = 'animate_no_context_experimental_kl'
    print(stats.f_oneway(*[df[df.verb_frequency == freq][col] for freq in ['HIGH', 'HIGH-MID', 'MID']]))
    print(stats.ttest_ind(df[df.verb_frequency == 'HIGH'][col], df[df.verb_frequency == 'HIGH-MID'][col]))
    print(stats.ttest_ind(df[df.verb_frequency == 'MID'][col], df[df.verb_frequency == 'HIGH-MID'][col]))
    print(stats.ttest_ind(df[df.verb_frequency == 'HIGH'][col], df[df.verb_frequency == 'MID'][col]))
    print(stats.ttest_ind(df[df.verb_category == 'physical'][col], df[df.verb_category == 'psychological'][col]))

    ###
    colorcycle = plt.rcParams['axes.prop_cycle'].by_key()['color']

    fig, axs_dict = plt.subplot_mosaic('D;E', sharex=True)
    axs = list(axs_dict.values())
    (ax1, ax2) = axs

    plot_df = verb_summary_df.copy()
    plot_df['Animacy Divergence'] = plot_df['animate_no_context_experimental_kl'] * 10000
    s1 = ax1.scatter(plot_df["Animacy Divergence"], np.zeros(len(plot_df)), alpha=0.25, c=colorcycle[0])

    plot_df = noun_summary_df.copy()
    plot_df['Animacy Divergence'] = plot_df['animate_no_context_experimental_kl'] * 10000
    s2 = ax2.scatter(plot_df["Animacy Divergence"], np.zeros(len(plot_df)), alpha=0.25, c=colorcycle[1])
    for k, ax in axs_dict.items():
        ax.text(0.02, 0.68, k, transform=ax.transAxes,
            size=20, weight='bold')
    fig.set_size_inches(5.66, 3)
    fig.supxlabel("Animacy Divergence (1e-4)")
    fig.suptitle("Mean Animacy Divergence per Verb and Noun")
    fig.legend([s1,s2], ['Verbs', 'Nouns'], loc=(0.8, 0.47), framealpha=1.0)
    fig.tight_layout()
    fig.show()
    if model == 'llama-7B':
        fig.savefig(f'paper-plots/scatters-{model}.pdf')
    fig.suptitle(f"Mean Animacy Divergence per Verb and Noun ({model})")
    #fig.savefig(f'paper-plots-png/scatters-{model}.png')
    fig.savefig(f'exp3-paper-plots/scatters-{model}.pdf')

# %%


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Apr  2 09:50:07 2019

@author: pickering
"""

import numpy as np
import scipy as sp
import matplotlib.pyplot as plt
import matplotlib.lines as lines
import matplotlib.gridspec as gs
import matplotlib.patches as patch
plt.rc('text', usetex=True)
plt.rc('font', family='sans-serif',size='12')

plt.close('all')
fig = plt.figure()
fig.set_size_inches(4,5)
ax = plt.gca()
plt.plot([0,1],[0.2,0.2], color='k')
plt.plot([0,1],[0.8,0.8], color='k')
plt.xlim(-0.5,1.5)
plt.ylim(0,1)
plt.xticks([])
plt.yticks([0.2,0.8],[r'$E_1$',r'$E_2'])
arrow = patch.FancyArrowPatch((0.1,0.2),(0.1,0.8),arrowstyle='simple',mutation_scale=20,color='b')
ax.add_artist(arrow)

arrow2 = patch.FancyArrowPatch((0.9,0.8),(0.9,0.2),arrowstyle='simple',mutation_scale=20,color='r')
ax.add_artist(arrow2)

plt.text(-0.2,0.25,r'Absorption',color='b',ha='center',va='center')
plt.text(1.2,0.25,r'Emission',color='r',ha='center',va='center')
plt.text(0.5,0.1,r'$h\nu = \Delta E = E_2-E_1$',color='k',ha='center',va='center')

arrow3 = patch.FancyArrowPatch((0.4,0.2),(0.4,0.8),arrowstyle='|-|',mutation_scale=5,color='k',ls='-',alpha=0.5)
ax.add_artist(arrow3)
plt.text(0.5,0.5,r'$h\nu$',color='k',ha='center',va='center')
plt.savefig('../final_figures/energy_gap.png', bbox_inches='tight', dpi=400)


#!/usr/bin/env python
"""
:Created: 2015-06-29

"""
from __future__ import print_function

__description__ = 'host -'
__version__ = '0.0.4-dev' # script-mpe
__db__ = '~/.taxus-code.sqlite'
__rc__ = '~/.domain.rc'
__usage__ = """

Usage:
  host.py [options] ( find <ref>...
                       | info [ <ref>... ]
                       | init [ -p... ]
                       | list
                       | new -p...
                       | update -p... <ref> )
  host.py [options] test init

Options:
    -c RC --config=RC
                  Use config file to load settings [default: %s]
    -d REF --dbref=REF
                  SQLAlchemy DB URL [default: %s].

Other flags:
    -h --help     Show this usage description.
                  For a command and argument description use the command 'help'.
    --version     Show version (%s).

""" % ( __rc__, __db__, __version__ )
__doc__ += __usage__

import os
from datetime import datetime

from script_mpe import log
from script_mpe import libcmd_docopt
from script_mpe.libcmd_docopt import cmd_help
from script_mpe.taxus import core, net, ScriptMixin
from script_mpe.domain2 import init_host

def cmd_init(settings):
    sa = net.Host.get_session('default', settings.dbref)
    host_dict = init_host(settings)
    name = host_dict['name']
    record = net.Host.fetch(filters=(net.Host.name == name,), sa=sa, exists=False)
    if not record:
        host = net.Host(name=name, date_added=datetime.now(),
                date_updated=datetime.now())
        sa.add(host)
        sa.commit()
        log.std('{bwhite}Added host %s record{default}', name)
    else:
        host = record
    print('host at', host_dict.path(), ':', host)

def cmd_list(settings):
    sa = net.Host.get_session('default', settings.dbref)
    for h in sa.query(net.Host).all():
        print(h)

def cmd_test_init(settings):
    sa = net.Host.get_session('default', settings.dbref)
    host_dict = init_host(settings)
    name = host_dict['name']
    print(net.Host.fetch(filters=(net.Host.name == name,), sa=sa, exists=False))
    #print net.Host.init(sa=sa)

### Transform cmd_ function names to nested dict

commands = libcmd_docopt.get_cmd_handlers(globals(), 'cmd_')
commands['help'] = libcmd_docopt.cmd_help

### Util functions to run above functions from cmdline

def main(opts):

    """
    Execute command.
    """

    #settings = opts.flags
    opts.flags.configPath = os.path.expanduser(opts.flags.config)
    settings = libcmd_docopt.init_config(opts.flags.configPath, dict(
            nodes = {}, interfaces = {}, domain = {}
        ), opts.flags)
    opts.default = 'info'
    return libcmd_docopt.run_commands(commands, settings, opts)

def get_version():
    return 'taxus-host-py.mpe/%s' % __version__

if __name__ == '__main__':
    import sys
    opts = libcmd_docopt.get_opts(__doc__, version=get_version())
    opts.flags.dbref = ScriptMixin.assert_dbref(opts.flags.dbref)
    sys.exit(main(opts))


# tests programmatic config of template system
import os

import pytest

BASE_DIR = os.path.dirname(__file__)

@pytest.fixture
def voila_config():
    def config(app):
        pass

    os.environ["JUPYTER_CONFIG_DIR"] = os.path.join(BASE_DIR, "../configs/general")
    yield config
    del os.environ["JUPYTER_CONFIG_DIR"]

@pytest.fixture
def voila_config_file_paths_arg():
    # we don't want the tests to use any configuration on the system
    path = os.path.abspath(os.path.join(BASE_DIR, "../configs/general"))
    return "--VoilaTest.config_file_paths=[%r]" % path

@pytest.mark.skip(reason="TODO: update for JupyterLab extensions")
async def test_lists_extension(http_server_client, base_url):
    response = await http_server_client.fetch(base_url)
    assert response.code == 200
    html_text = response.body.decode("utf-8")
    assert "Hi Voilà" in html_text
    assert "ipytest/extension.js" in html_text


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

from sqlalchemy import Column
from sqlalchemy import ForeignKey
from sqlalchemy import MetaData
from sqlalchemy import String
from sqlalchemy import Table
from sqlalchemy import create_engine
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import relationship
from sqlalchemy.orm import sessionmaker

# 初始化数据连接,create_engine()用来初始化数据库连接
# SQLAlchemy用一个字符串表示连接信息：'数据库类型+数据库驱动名称://用户名:口令@机器地址:端口号/数据库名'
engine = create_engine('mysql+mysqlconnector://root:root@localhost:3306/test_python')

# 创建元数据
metadata = MetaData(engine)

# 创建表
c = Table('classroom', metadata,
      Column('id', String(20), primary_key=True),
      Column('name', String(20)),
      mysql_engine='InnoDB',
      mysql_charset='utf8',
      mysql_key_block_size="1024"
     )
t = Table('teacher', metadata,
      Column('id', String(20), primary_key=True),
      Column('name', String(20)),
      Column('subject', String(20)),
      Column('class_id', String(20)),
      mysql_engine='InnoDB',
      mysql_charset='utf8',
      mysql_key_block_size="1024"
     )
metadata.create_all()

# 创建对象基类
Base = declarative_base()

# 班级
class Classroom(Base):
    __tablename__ = 'classroom'

    id = Column(String(20), primary_key=True)
    name = Column(String(20))
    # 一对多
    teacher = relationship('Teacher')

    def __repr__(self):
        return '<User: (id: %s, name: %s, teacher: %s)>' % (self.id, self.name, self.teacher)

class Teacher(Base):
    __tablename__ = 'teacher'

    id = Column(String(20), primary_key=True)
    name = Column(String(20))
    subject = Column(String(20))

    # "多"的一方teacher表是通过外键关联到class表
    class_id = Column(String(20), ForeignKey('classroom.id'))

    def __repr__(self):
        return '<Teacher: (id: %s, name: %s, subject: %s, class_id: %s)>' % (self.id, self.name, self.subject, self.class_id)

# 多次插入方式
def insert_multiple():
    new_class = Classroom(id='1', name='三年二班')
    session.add(new_class)
    session.commit()

    math_teacher = Teacher(id='1', name='李志', class_id='1', subject='数学')
    chiese_teacher = Teacher(id='2', name='赵雷', class_id='1', subject='语文')
    session.add_all([math_teacher, chiese_teacher])
    session.commit()

# 一次插入方式
def insert_once():
    math_teacher = Teacher(id='1', name='李志', class_id='1', subject='数学')
    chiese_teacher = Teacher(id='2', name='赵雷', class_id='1', subject='语文')
    new_class = Classroom(id='1', name='三年二班', teacher=[math_teacher, chiese_teacher])
    session.add(new_class)
    session.commit()

DBsession = sessionmaker(bind=engine)
session = DBsession()

insert_once()

result = session.query(Classroom).filter(Classroom.id == '1').all()
print(result)

session.close()


"""
Graphene is based on an octree data structure. We mesh at
the lowest agglomerated level of the octree (L2) and then
create stitched meshes at each higher level of the tree
recursively starting from those L2 meshes. This scheme allows
use to rapidly reconfigure small regions that change while
proofreading.

The goal of meshing is to render each L2 label into a distinct
mesh. However, in order for the stitching to work, as in a flat
segmentation, we need to extend one voxel into the neighboring
chunk.

Each L2 chunk represents a self contained proofread agglomeration
of the L1 watershed regions over an area of typically 256x256x512
voxels. Actual proofreading extends across L2 chunks, but this is
represented by higher levels of the octree. Therefore, adjacent
L2 chunks have completely different labels.

In order to create accurate merging between chunks, we'll need to
relabel the overlap region using the lowest parent chunk ids between
the primary chunk and each individual overlap region (therefore each
overlap region may require a different chunk graph level). The reason
we can't just label them all with the root ID and mesh is that levels
higher than the lowest level may introduce non-local information. For
instance, a self touch on the boundary that would have been kept apart
at L3 would be unified at the root. The chunk position determines what
the lowest root is. For example, a chunk located at the origin will
require an L3 overlap merge. A chunk located one unit from the origin
will require an L4 merge. A chunk located three units from the origin
will require an L5 merge, and so on according to the fan-out of the octree.

Once the chunk is relabeled using the correct parent, some odd cases
crop up.  For example, due to information in the overlap region, L2
labels that were formerly independent connected components may be
merged via a chain of one or more L3+ relabeled overlap labels. Therefore,
we apply the following rules:

1. If the L2 label does not touch the overlap region, use that label.
2. Each overlap supervoxel is merged with neighboring labels that share
  a parent. Each resulting connected component is remapped using the minimum
  covered L2 label from the primary chunk. If no such label exists it
  is mapped to 0.

"""
from typing import Sequence
import numpy as np
import time
import collections
from functools import lru_cache
import datetime
import pytz

import networkx as nx

import cc3d

from tqdm import tqdm

import cloudvolume
from cloudvolume.lib import Bbox, Vec, xyzrange
from cloudvolume import CloudVolume
import fastremap

UTC = pytz.UTC

def remap_segmentation(
  cv, chunk_x, chunk_y, chunk_z, mip=2,
  overlap_vx=1, time_stamp=None, progress=False
):
  ws_cv = CloudVolume(
    cv.meta.cloudpath, mip=mip,
    progress=progress, fill_missing=cv.fill_missing
  )
  mip_diff = mip - cv.meta.watershed_mip

  mip_chunk_size = np.array(
    cv.meta.graph_chunk_size, dtype=np.int
  ) / np.array([2 ** mip_diff, 2 ** mip_diff, 1])
  mip_chunk_size = mip_chunk_size.astype(np.int)

  offset = Vec(chunk_x, chunk_y, chunk_z) * mip_chunk_size
  bbx = Bbox(offset, offset + mip_chunk_size + overlap_vx)
  if cv.meta.chunks_start_at_voxel_offset:
    bbx += ws_cv.voxel_offset
  bbx = Bbox.clamp(bbx, ws_cv.bounds)

  seg = ws_cv[bbx][..., 0]

  if not np.any(seg):
    return seg

  sv_remapping, unsafe_dict = get_lx_overlapping_remappings(
    cv, chunk_x, chunk_y, chunk_z,
    time_stamp=time_stamp, progress=progress
  )

  seg = fastremap.mask_except(seg, list(sv_remapping.keys()), in_place=True)
  fastremap.remap(seg, sv_remapping, preserve_missing_labels=True, in_place=True)

  for unsafe_root_id in tqdm(unsafe_dict.keys(), desc="Unsafe Relabel", disable=(not progress)):
    bin_seg = seg == unsafe_root_id

    if np.sum(bin_seg) == 0:
        continue

    cc_seg = cc3d.connected_components(bin_seg)
    for i_cc in range(1, np.max(cc_seg) + 1):
      bin_cc_seg = cc_seg == i_cc

      overlaps = []
      overlaps.extend(np.unique(seg[-2, :, :][bin_cc_seg[-1, :, :]]))
      overlaps.extend(np.unique(seg[:, -2, :][bin_cc_seg[:, -1, :]]))
      overlaps.extend(np.unique(seg[:, :, -2][bin_cc_seg[:, :, -1]]))
      overlaps = np.unique(overlaps)

      linked_l2_ids = overlaps[np.in1d(overlaps, unsafe_dict[unsafe_root_id])]

      if len(linked_l2_ids) == 0:
        seg[bin_cc_seg] = 0
      else:
        seg[bin_cc_seg] = linked_l2_ids[0]

  return seg

@lru_cache(maxsize=200)
def get_higher_to_lower_remapping(cv, chunk_id, time_stamp):
  """
  Retrieves lx node id to sv id mappping

  :param cv: CloudVolumeGraphene object
  :param chunk_id: np.uint64
  :param time_stamp: datetime object
  :return: dictionary
  """
  rr_chunk = cv.get_chunk_mappings(chunk_id=chunk_id, timestamp=time_stamp)

  # This for-loop ensures that only the latest lx_ids are considered
  # The order by id guarantees the time order (only true for same neurons
  # but that is the case here).
  lx_remapping = {}
  all_lower_ids = set()
  for k in sorted(rr_chunk.keys(), reverse=True):
    this_child_ids = rr_chunk[k]

    if this_child_ids[0] in all_lower_ids:
      continue

    all_lower_ids.update(list(this_child_ids))
    lx_remapping[k] = this_child_ids

  return lx_remapping

@lru_cache(maxsize=None)
def get_root_lx_remapping(cv, chunk_id, stop_layer, time_stamp):
  """
  Retrieves root to l2 node id mapping

  :param cv: CloudVolumeGraphene object
  :param chunk_id: np.uint64
  :param stop_layer: int
  :param time_stamp: datetime object
  :return: multiples
  """
  lx_id_remap = get_higher_to_lower_remapping(
    cv, chunk_id, time_stamp=time_stamp
  )

  lx_ids = np.array(list(lx_id_remap.keys()))
  root_ids = cv.get_roots(lx_ids, stop_layer=stop_layer)

  return lx_ids, np.array(root_ids), lx_id_remap

# @lru_cache(maxsize=None)
def get_lx_overlapping_remappings(
  cv, chunk_x, chunk_y, chunk_z,
  time_stamp=None, progress=False
):
  """
  Retrieves sv id to layer mapping for chunk with overlap in positive
    direction (one chunk)

  :param cv: CloudVolumeGraphene object
  :param chunk_x: np.uint64
  :param chunk_y: np.uint64
  :param chunk_z: np.uint64
  :param time_stamp: datetime object
  :return: multiples
  """
  if time_stamp is None:
    time_stamp = datetime.datetime.utcnow()

  if isinstance(time_stamp, datetime.datetime) and time_stamp.tzinfo is None:
    time_stamp = UTC.localize(time_stamp)

  chunk_layer = 2
  chunk_id = cv.meta.encode_label(chunk_layer, chunk_x, chunk_y, chunk_z, 0)

  neigh_chunk_ids = []
  neigh_parent_chunk_ids = []

  # Collect neighboring chunks and their parent chunk ids
  # We only need to know about the parent chunk ids to figure the lowest
  # common chunk
  # Notice that the first neigh_chunk_id is equal to `chunk_id`.
  for x in range(chunk_x, chunk_x + 2):
    for y in range(chunk_y, chunk_y + 2):
      for z in range(chunk_z, chunk_z + 2):

        neigh_chunk_id = cv.meta.encode_label(layer=2, x=x, y=y, z=z, segid=0)
        neigh_chunk_ids.append(neigh_chunk_id)
        neigh_parent_chunk_ids.append(
          get_parent_chunk_ids(cv, neigh_chunk_id)
        )

  # Find lowest common chunk
  # finding the layer that has the lowest common parent of
  # these 8 neighboring chunks
  # Layer agreement will equal 0 if that is layer 3, 1 if
  # that is layer 4, etc
  neigh_parent_chunk_ids = np.array(neigh_parent_chunk_ids)
  layer_agreement = np.all(
    (neigh_parent_chunk_ids - neigh_parent_chunk_ids[0]) == 0, axis=0
  )
  stop_layer = np.where(layer_agreement)[0][0] + chunk_layer

  # Find the parent in the lowest common chunk for each l2 id. These parent
  # ids are referred to as root ids even though they are not necessarily the
  # root id.
  neigh_lx_ids = []
  neigh_lx_id_remap = {}
  neigh_root_ids = []

  safe_lx_ids = []
  unsafe_lx_ids = []
  unsafe_root_ids = []

  # This loop is the main bottleneck
  for neigh_chunk_id in tqdm(neigh_chunk_ids, disable=(not progress), desc="Neighbor"):
    # print(f"Neigh: {neigh_chunk_id} --------------")

    lx_ids, root_ids, lx_id_remap = get_root_lx_remapping(
      cv, neigh_chunk_id, stop_layer, time_stamp=time_stamp
    )
    neigh_lx_ids.extend(lx_ids)
    neigh_lx_id_remap.update(lx_id_remap)
    neigh_root_ids.extend(root_ids)

    if neigh_chunk_id == chunk_id:
      # The first neigh_chunk_id is the one we are interested in. All lx
      # ids that share no root id with any other lx id are "safe", meaning
      # that we can easily obtain the complete remapping (including
      # overlap) for these. All other ones have to be resolved using the
      # segmentation.
      _, u_idx, c_root_ids = np.unique(
        neigh_root_ids, return_counts=True, return_index=True
      )

      safe_lx_ids = lx_ids[u_idx[c_root_ids == 1]]
      unsafe_lx_ids = lx_ids[~np.in1d(lx_ids, safe_lx_ids)]
      unsafe_root_ids = np.unique(root_ids[u_idx[c_root_ids != 1]])

  lx_root_dict = dict(zip(neigh_lx_ids, neigh_root_ids))
  root_lx_dict = collections.defaultdict(list)

  # Future sv id -> lx mapping
  sv_ids = []
  lx_ids_flat = []

  # Do safe ones first
  for i_root_id in range(len(neigh_root_ids)):
    root_lx_dict[neigh_root_ids[i_root_id]].append(neigh_lx_ids[i_root_id])

  for lx_id in safe_lx_ids:
    root_id = lx_root_dict[lx_id]
    for neigh_lx_id in root_lx_dict[root_id]:
      lx_sv_ids = neigh_lx_id_remap[neigh_lx_id]
      sv_ids.extend(lx_sv_ids)
      lx_ids_flat.extend([lx_id] * len(neigh_lx_id_remap[neigh_lx_id]))

  # For the unsafe ones we can only do the in chunk svs
  # But we will map the out of chunk svs to the root id and store the
  # hierarchical information in a dictionary
  for lx_id in unsafe_lx_ids:
    sv_ids.extend(neigh_lx_id_remap[lx_id])
    lx_ids_flat.extend([lx_id] * len(neigh_lx_id_remap[lx_id]))

  unsafe_dict = collections.defaultdict(list)
  for root_id in unsafe_root_ids:
    if np.sum(~np.in1d(root_lx_dict[root_id], unsafe_lx_ids)) == 0:
      continue

    for neigh_lx_id in root_lx_dict[root_id]:
      unsafe_dict[root_id].append(neigh_lx_id)

      if neigh_lx_id in unsafe_lx_ids:
        continue

      sv_ids.extend(neigh_lx_id_remap[neigh_lx_id])
      lx_ids_flat.extend([root_id] * len(neigh_lx_id_remap[neigh_lx_id]))

  # Combine the lists for a (chunk-) global remapping
  sv_remapping = dict(zip(sv_ids, lx_ids_flat))

  return sv_remapping, unsafe_dict

def get_parent_chunk_ids(cv, label):
  """
  Creates list of chunk parent ids

  :param label: np.uint64
  :return: np.ndarray
  """
  parent_chunk_layers = range(
    int(cv.meta.decode_layer_id(label)) + 1, int(cv.meta.n_layers) + 1
  )

  chunk_coord = Vec(*cv.meta.decode_chunk_position(label))

  parent_chunk_ids = [ cv.meta.decode_chunk_id(label) ]
  for layer in parent_chunk_layers:
    chunk_coord = chunk_coord // cv.meta.fan_out
    chunk_id = cv.meta.encode_label(
      layer, chunk_coord[0], chunk_coord[1], chunk_coord[2], segid=0
    )
    parent_chunk_ids.append(chunk_id)

  return np.array(parent_chunk_ids, dtype=np.uint64)


import random
import torch
from utils import print_rank_0
from SwissArmyTransformer.model import GLMModel, EncoderDecoderModel
from SwissArmyTransformer.model.mixins import BaseMixin
from SwissArmyTransformer.model.base_model import non_conflict
from SwissArmyTransformer.mpu.transformer import standard_attention, split_tensor_along_last_dim

def get_prefix_model(model_cls):
    def new_model_cls(args):
        model = model_cls(args)
        if args.freeze_transformer:
            print_rank_0("Freeze transformer parameters")
            model.requires_grad_(False)
        prefix_mixin = PrefixPromptMixin(prefix_length=args.prefix_prompt, num_layers=args.num_layers,
                                         hidden_size=args.hidden_size, prompt_func=args.prompt_func)
        if isinstance(model, GLMModel):
            model.add_mixin(
                "prefix_prompt",
                prefix_mixin
            )
        elif isinstance(model, EncoderDecoderModel):
            model.encoder.add_mixin(
                "prefix_prompt",
                prefix_mixin
            )
        else:
            raise NotImplementedError(model_cls)
        return model

    return new_model_cls

class PrefixPromptMixin(BaseMixin):
    def __init__(self, prefix_length, num_layers, hidden_size, prompt_func='none'):
        super().__init__()
        self.prefix_length = prefix_length
        self.num_layers = num_layers
        self.hidden_size = hidden_size
        self.embeddings = torch.nn.ModuleList(
            [torch.nn.Embedding(prefix_length, hidden_size) for _ in range(self.num_layers)]
        )
        for embedding in self.embeddings:
            torch.nn.init.normal_(embedding.weight, mean=0, std=0.1)
        self.prompt_func = prompt_func
        if prompt_func != 'none':
            raise NotImplementedError(prompt_func)

    def get_prompt(self, batch_size, layer_id):
        prefix_hidden_states = self.embeddings[layer_id].weight
        prefix_hidden_states = prefix_hidden_states.unsqueeze(0).expand(batch_size, -1, -1)
        return prefix_hidden_states

    @non_conflict
    def attention_fn(self, q, k, v, mask, dropout_fn, old_impl=standard_attention, layer_id=None, cross_attention=False,
                     **kw_args):
        if not cross_attention:
            attn_module = self.transformer.layers[layer_id].attention
            prefix = self.get_prompt(q.size(0), layer_id)
            mixed_raw_layer = attn_module.query_key_value(prefix)
            (_,
             mixed_key_prefix,
             mixed_value_prefix) = split_tensor_along_last_dim(mixed_raw_layer, 3)
            key_prefix = attn_module._transpose_for_scores(mixed_key_prefix)
            value_prefix = attn_module._transpose_for_scores(mixed_value_prefix)
            k, v = torch.cat((key_prefix, k), dim=2), torch.cat((value_prefix, v), dim=2)
            if mask.numel() > 1:
                mask_prefix = torch.ones(self.prefix_length, device=mask.device, dtype=mask.dtype)
                mask_prefix = mask_prefix.expand(*(mask.size()[:-1]), -1)
                mask = torch.cat((mask_prefix, mask), dim=-1)
        return old_impl(q, k, v, mask, dropout_fn, layer_id=layer_id, cross_attention=cross_attention,
                        **kw_args)


import matplotlib.pyplot as plt

import pathlib

text = pathlib.Path("test.txt").read_text()

rows = []
for line in text.splitlines():
    rows.append([float(val) for val in line.split()])

import numpy as np

data = np.array(rows)
print(data.shape)
plt.matshow(data)
plt.show()


from psycopg_pool import ConnectionPool
import os

try:
    pool = ConnectionPool(conninfo=os.environ["DATABASE_URL"])
except KeyError:
    pool = None

class CurrencyQueries:
    def structure_countries(self, country: list):
        try:
            return {
                "id": country[0],
                "currency_code": country[1],
                "country": country[2],
            }
        except IndexError:
            return {"warning": "impropertly formatted country."}

    def get_list_from_db(self):
        """
        Arguments: N/A
        Returns: a list containing data about each country, received
        from the database
        """
        with pool.connection() as conn:
            with conn.cursor() as cur:
                cur.execute(
                    """
                        SELECT id
                        , currency_code
                        , country
                        FROM currency;
                    """
                )
                results = []
                countries = cur.fetchall()
                for country in countries:
                    formatted_country = self.structure_countries(country)
                    if "warning" in formatted_country:
                        return []
                    results.append(formatted_country)
                return results

    def get_currency_code_pair(self, origin_country, destination_country):
        with pool.connection() as conn:
            with conn.cursor() as cur:
                cur.execute(
                    """
                        SELECT currency_code
                        FROM currency
                        WHERE %s = country
                    """,
                    [origin_country],
                )
                origin_code = cur.fetchall()[0][0]
                cur.execute(
                    """
                        SELECT currency_code
                        FROM currency
                        WHERE %s = country
                    """,
                    [destination_country],
                )
                destination_code = cur.fetchall()[0][0]
                return [origin_code, destination_code]


#first, I had to do this in the console $ python3 -m pip install boto3
#also make sure amazon CLI is set up.
import boto3

#ec2 stuff
print("*****Printing describe instances of EC2 Instances\n")
ec2 = boto3.client('ec2')

response = ec2.describe_instances()
for reservation in response['Reservations']:
    for instance in reservation['Instances']:
        print(instance, '\n')
# write an output file with entire response.  This will overwrite anything previous.
f=open("./Results/ec2_descriptions.json", "w+")
f.write("{'Results of ec2.describe_instances in boto3': %s}" % (response))
f.close()
print("File ec2_descriptions.json was created or updated in the Results folder.")

print("Open ec2_descriptions.json in Results folder to see")


import os
from setuptools import setup

def read(fname):
    return

setup(
    name = 'Nosferatu',
    version = '0.4.4',
    author = 'Daniel Cardin',
    author_email = 'ddcardin@gmail.com',
    description = (
        'A webapp to remotely control your home lights and wall sockets'
    ),
    license = 'Apache 2.0',
    keywords = 'nosferatu webapp home automation lights lightswitch',
    packages=['nosferatu', 'tests'],
    long_description=open(os.path.join(os.path.dirname(__file__), 'README.md')).read(),
    classifiers=[
        'Development Status :: 3 - Alpha',
        'Topic :: Internet :: WWW/HTTP :: WSGI :: Application',
        'License :: OSI Approved :: Apache Software License',
        'Programming Language :: Python :: 3',
    ],
    install_requires=[
        'APScheduler',
        'alembic',
        'cssmin',
        'Flask',
        'Flask-Assets',
        'Flask-User',
        'Flask-Cache',
        'Flask-Login',
        'Flask-Migrate',
        'Flask-SQLAlchemy',
        'SQLAlchemy',
        'gunicorn',
        'jsmin',
        'psycopg2',
        'redis',
        'webassets',
    ],
)


#!/usr/bin/env python3
"""
    CANReverse CAN Bus Reverse-Engineering-Software as part of my bachelors thesis
    "Conception of a prototypical realization of selected attacks on the CAN bus via OBD-II".

    (c) Jannik Schmied, 2023
"""
import sys

from argparse import ArgumentParser, Namespace
from datetime import datetime
from shlex import split
from subprocess import Popen, DEVNULL, PIPE
from time import sleep

from pyfiglet import figlet_format

# Show extended error messages
DEBUG: bool = False

# Constants
VERSION: str = "1.0.0rc1"
FILE: str
SANITIZE: bool = False
SILENT: bool = False
INTERFACE: str = "vcan0"

def parse_args() -> Namespace:
    parser: ArgumentParser = ArgumentParser()
    parser.add_argument("--file", "-f", help="Specify candump logfile (needs to be sanitized, use --sanitize/-s else)", type=str, required=True)
    parser.add_argument("--sanitize", "-s", help="Sanitizes file for usage", action='store_true')
    parser.add_argument("--silent", help="Suppress unnecessary output", action='store_true')
    parser.add_argument("--interface", "-i", help="Specify CAN interface (default: vcan0)", type=str, default="vcan0")

    return parser.parse_args()

def sanitize(file: str) -> str:
    new_file: str = f"candump_logfile_{datetime.now().strftime('%Y_%m_%d-%H_%M_%S')}_sanitized.log"
    command: str = f"cat {file}" + " | awk '{print $3}' | sort | uniq" + f" > {new_file}"

    try:
        sanitize_file_process = Popen(split(command), stdout=DEVNULL, stderr=PIPE)
        sanitize_file_process.terminate()
    except Exception as e:
        print("[!] An error occurred:", e)
        sys.exit(1)

    return new_file

def main():
    args = parse_args()

    global FILE
    FILE = args.file

    if args.sanitize:
        global SANITIZE
        SANITIZE = args.sanitize
    if args.silent:
        global SILENT
        SILENT = args.silent
    if args.interface:
        global INTERFACE
        INTERFACE = args.interface

    file: str = FILE

    if not SILENT:
        print("\n", figlet_format("CANREVERSE"))
        print(f"(c) Jannik Schmied, 2022. Version {VERSION}")
        print("-" * 56)

    if SANITIZE:
        print(f"[*] Sanitizing file for further usage...")
        file = sanitize(file)

    try:
        print("[*] Reading file...")
        with open(file, "r") as logfile:
            lines = logfile.readlines()
            packet_counter: int = 0
            for line in lines:
                packet_counter += 1
            send_counter: int = 0

            print("[*] Start reverse engineering process")
            print(f"[i] Total packets: {packet_counter}")
            for line in lines:
                can_message = line
                command = f"cansend {INTERFACE} {can_message}"

                can_message = can_message.replace('\n', '').replace('\r', '')
                print(f"[*] Sending packet {can_message} ({send_counter}/{packet_counter})")
                for counter in range(1):
                    try:
                        send_can_message_process = Popen(split(command), stdout=PIPE, stderr=PIPE)
                        send_counter += 1
                        send_can_message_process.terminate()
                    except Exception as e:
                        print(f"[!] Error: {e}")
                        exit(1)
                sleep(3)
    except FileExistsError:
        print("[!] File not found!")
        sys.exit(1)
    except FileNotFoundError:
        print("[!] File not found!")
        sys.exit(1)
    except KeyboardInterrupt:
        print("\n[i] Stopped. (Interrupted by user)")
    except Exception as e:
        print(f"[!] Error: {e}")
    finally:
        print(f"[i] Finished! Sent {send_counter} of {packet_counter} packets.")

if __name__ == '__main__':
    main()


import warnings
from enum import IntFlag
from typing import Optional, Union

from .. import xdr as stellar_xdr
from ..asset import Asset
from ..keypair import Keypair
from ..muxed_account import MuxedAccount
from ..strkey import StrKey
from ..utils import raise_if_not_valid_ed25519_public_key
from .operation import Operation

__all__ = ["TrustLineEntryFlag", "AllowTrust"]

class TrustLineEntryFlag(IntFlag):
    """Indicates which flags to set. For details about the flags,
    please refer to the `CAP-0018 <https://github.com/stellar/stellar-protocol/blob/master/core/cap-0018.md>`_.

    - **UNAUTHORIZED_FLAG**: The account can hold a balance but cannot receive payments, send payments, maintain offers or manage offers

    - **AUTHORIZED_FLAG**: The account can hold a balance, receive payments, send payments, maintain offers or manage offers

    - **AUTHORIZED_TO_MAINTAIN_LIABILITIES_FLAG**: The account can hold a balance and maintain offers but cannot receive payments, send payments or manage offers
    """

    UNAUTHORIZED_FLAG = 0
    AUTHORIZED_FLAG = 1
    AUTHORIZED_TO_MAINTAIN_LIABILITIES_FLAG = 2

class AllowTrust(Operation):
    """The :class:`AllowTrust` object, which represents a AllowTrust operation
    on Stellar's network.

    Updates the authorized flag of an existing trustline. This can only be
    called by the issuer of a trustline's `asset
    <https://developers.stellar.org/docs/glossary/assets/>`_.

    The issuer can only clear the authorized flag if the issuer has the
    ``AUTH_REVOCABLE_FLAG`` set. Otherwise, the issuer can only set the authorized
    flag.

    Threshold: Low

    See `Allow Trust <https://developers.stellar.org/docs/start/list-of-operations/#allow-trust>`_ for more information.

    :param trustor: The trusting account (the one being authorized).
    :param asset_code: The asset code being authorized.
    :param authorize: `True` to authorize the line, `False` to deauthorize，if you need further control,
        you can also use :class:`stellar_sdk.operation.allow_trust.TrustLineEntryFlag`.
    :param source: The source account for the operation. Defaults to the transaction's source account.

    """

    _XDR_OPERATION_TYPE: stellar_xdr.OperationType = (
        stellar_xdr.OperationType.ALLOW_TRUST
    )

    def __init__(
        self,
        trustor: str,
        asset_code: str,
        authorize: Union[TrustLineEntryFlag, bool],
        source: Optional[Union[MuxedAccount, str]] = None,
    ) -> None:
        # We keep this class to ensure that the SDK can correctly parse historical transactions.
        warnings.warn(
            "Use `stellar_sdk.operation.set_trust_line_flags.SetTrustLineFlags` instead.",
            DeprecationWarning,
        )
        super().__init__(source)

        self.trustor: str = trustor
        self.asset_code: str = asset_code

        if isinstance(authorize, bool):
            if authorize is True:
                self.authorize: TrustLineEntryFlag = TrustLineEntryFlag.AUTHORIZED_FLAG
            else:
                self.authorize = TrustLineEntryFlag.UNAUTHORIZED_FLAG
        else:
            self.authorize = authorize
        raise_if_not_valid_ed25519_public_key(self.trustor, "trustor")
        Asset.check_if_asset_code_is_valid(self.asset_code)

    def _to_operation_body(self) -> stellar_xdr.OperationBody:
        trustor = Keypair.from_public_key(self.trustor).xdr_account_id()
        length = len(self.asset_code)
        pad_length = 4 - length if length <= 4 else 12 - length
        asset_code = bytearray(self.asset_code, "ascii") + b"\x00" * pad_length
        authorize = stellar_xdr.Uint32(self.authorize.value)
        if len(asset_code) == 4:
            asset_type = stellar_xdr.AssetType.ASSET_TYPE_CREDIT_ALPHANUM4
            asset_code4 = stellar_xdr.AssetCode4(asset_code)
            asset = stellar_xdr.AssetCode(type=asset_type, asset_code4=asset_code4)
        else:
            asset_type = stellar_xdr.AssetType.ASSET_TYPE_CREDIT_ALPHANUM12
            asset_code12 = stellar_xdr.AssetCode12(asset_code)
            asset = stellar_xdr.AssetCode(type=asset_type, asset_code12=asset_code12)
        allow_trust_op = stellar_xdr.AllowTrustOp(trustor, asset, authorize)
        body = stellar_xdr.OperationBody(
            type=self._XDR_OPERATION_TYPE, allow_trust_op=allow_trust_op
        )
        return body

    @classmethod
    def from_xdr_object(cls, xdr_object: stellar_xdr.Operation) -> "AllowTrust":
        """Creates a :class:`AllowTrust` object from an XDR Operation
        object.

        """
        source = Operation.get_source_from_xdr_obj(xdr_object)
        assert xdr_object.body.allow_trust_op is not None
        assert xdr_object.body.allow_trust_op.trustor.account_id.ed25519 is not None
        trustor = StrKey.encode_ed25519_public_key(
            xdr_object.body.allow_trust_op.trustor.account_id.ed25519.uint256
        )
        authorize = xdr_object.body.allow_trust_op.authorize.uint32
        authorize = TrustLineEntryFlag(authorize)
        asset_type = xdr_object.body.allow_trust_op.asset.type
        if asset_type == stellar_xdr.AssetType.ASSET_TYPE_CREDIT_ALPHANUM4:
            assert xdr_object.body.allow_trust_op.asset.asset_code4 is not None
            asset_code = (
                xdr_object.body.allow_trust_op.asset.asset_code4.asset_code4.decode()
            )
        elif asset_type == stellar_xdr.AssetType.ASSET_TYPE_CREDIT_ALPHANUM12:
            assert xdr_object.body.allow_trust_op.asset.asset_code12 is not None
            asset_code = (
                xdr_object.body.allow_trust_op.asset.asset_code12.asset_code12.decode()
            )
        else:
            raise NotImplementedError(
                "Operation of asset_type={} is not implemented" ".".format(asset_type)
            )

        asset_code = asset_code.rstrip("\x00")
        op = cls(
            source=source, trustor=trustor, authorize=authorize, asset_code=asset_code
        )
        return op

    def __str__(self):
        return (
            f"<AllowTrust [trustor={self.trustor}, "
            f"asset_code={self.asset_code}, "
            f"authorize={self.authorize}, "
            f"source={self.source}]>"
        )


# -*- coding: utf-8 -*-
"""
UNIVERSIDADE FEDERAL RURAL DE PERNAMBUCO - UFRPE
UNIDADE ACADÊMICA DO CABO DE SANTO AGOSTINHO - UACSA

@author(s):
    Rogério Soares (rogerio.soaress@ufrpe.br)

Created on Wed Apr 29 00:03:47 2020
"""

import tkinter
from matplotlib.backends.backend_tkagg import (FigureCanvasTkAgg, NavigationToolbar2Tk)
# Implement the default Matplotlib key bindings.
from matplotlib.backend_bases import key_press_handler
from matplotlib.figure import Figure
import numpy as np

def on_key_press(event):
    print("you pressed {}".format(event.key))
    key_press_handler(event, canvas, toolbar)

canvas.mpl_connect("key_press_event", on_key_press)

def _quit():
    root.quit()     # stops mainloop
    root.destroy()  # this is necessary on Windows to prevent
                    # Fatal Python Error: PyEval_RestoreThread: NULL tstate

def plot_stat(window):

    global root
    root = tkinter.Toplevel(window)
    root.wm_title("Embedding in Tk")

    fig = Figure(figsize=(5, 4), dpi=100)
    t = np.arange(0, 3, .01)
    fig.add_subplot(111).plot(t, 2 * np.sin(2 * np.pi * t))

    global canvas
    canvas = FigureCanvasTkAgg(fig, master=root)  # A tk.DrawingArea.
    canvas.draw()
    canvas.get_tk_widget().pack(side=tkinter.TOP, fill=tkinter.BOTH, expand=1)

    global toolbar
    toolbar = NavigationToolbar2Tk(canvas, root)
    toolbar.update()
    canvas.get_tk_widget().pack(side=tkinter.TOP, fill=tkinter.BOTH, expand=1)

    button = tkinter.Button(master=root, text="Quit", command=_quit)
    button.pack(side=tkinter.BOTTOM)


# Author: Mattia Silvestri

"""
Main program.
"""

import os

os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
from utility import PLSInstance, PLSSolver, random_assigner, from_one_hot_to_2d, from_2d_to_one_hot
from models import MyModel
import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow.keras.layers import Dense, Conv2D, Flatten
import csv
import argparse
import time
import pandas as pd

########################################################################################################################

# Set seed in order to reproduce results
tf.random.set_seed(0)

# Tensorflow 2 GPU setup
gpus = tf.config.experimental.list_physical_devices('GPU')
if gpus:
    # Restrict TensorFlow to only use the first GPU
    try:
        tf.config.experimental.set_visible_devices(gpus[0], 'GPU')
        tf.config.experimental.set_virtual_device_configuration(
            gpus[0],
            [tf.config.experimental.VirtualDeviceConfiguration(memory_limit=8192)])
        logical_gpus = tf.config.experimental.list_logical_devices('GPU')
        print(len(gpus), "Physical GPUs,", len(logical_gpus), "Logical GPU")
    except RuntimeError as e:
        # Visible devices must be set before GPUs have been initialized
        print(e)

########################################################################################################################

parser = argparse.ArgumentParser()
parser.add_argument("--dim", type=int, default=10, help="Problem dimension")
parser.add_argument("--train", action="store_true",
                    help="Train the model; if not set the default is test mode.", default=False)
parser.add_argument("--test-num", default=None,
                    help="Test identifier.")
parser.add_argument("--num-epochs", default=300, type=int,
                    help="Number of training epochs.")
parser.add_argument("--max-size", default=1000000, type=int,
                    help="Maximum number of training/test instances to be loaded.")
parser.add_argument("--load-mode", default="onehot", choices=["onehot", "string"],
                    help="Dataset loading mode.")
parser.add_argument("--batch-size", default=1024, type=int,
                    help="Mini-batch size.")
parser.add_argument("--leave-columns-domains", action="store_true", default=False,
                    help="True if you don't want to prune columns domains values with forward checking.")
parser.add_argument("--num-sol", type=str, default="10k",
                    help="Number of solutions from which the training set has been generated; thousands are expressed "
                         + "with k (for example 10000=10k).")
parser.add_argument("--model", choices=["fnn", "cnn"], required=True,
                    help="Choose the model architecture.")  # TODO: change default
parser.add_argument("--model-type", default="agnostic", choices=["agnostic", "sbrinspiredloss", "negative", "binary"],
                    help="Choose the model type. 'agnostic' is the model-agnostic baseline. 'sbrinspiredloss', "
                         + "'negative' and 'binary' are relatively the mse, negative and binary-cross entropy versions"
                         + " of the SBR inspiredloss.")
parser.add_argument("--validation-size", type=int, default=0,
                    help="Validation set dimension. If zero no validation set is used.")
parser.add_argument("--use-prop", action="store_true", default=False,
                    help="True if you want to assist the estimators with constraints propagation at evaluation time.")
parser.add_argument("--rnd-feas", action="store_true", default=False,
                    help="True if you want to compute feasibility ratio also for random estimator.")
parser.add_argument("--lmbd", default=1.0, type=float, help="Lambda for SBR-inspired term.")
parser.add_argument("--patience", default=10, type=int,
                    help="Specify the number of 10 epochs intervals without improvement in "
                         "feasibility after which training is stopped.")

args = parser.parse_args()
print(args)

# Problem dimension.
DIM = int(args.dim)

COLUMN_TYPES = [int() for _ in range(DIM ** 3)]

# Set training or test mode.
TRAIN = args.train
if TRAIN:
    print("Training mode")
    mode = "train"
else:
    print("Test mode")
    mode = "test"

# Test number identifier
TEST_NUM = args.test_num

# Number of training epochs
EPOCHS = int(args.num_epochs)

# Maximum number of data set examples to load
MAX_SIZE = int(args.max_size)

# Available loading mode are string and one-hot
LOAD_MODE = args.load_mode

# Mini-batch size
BATCH_SIZE = int(args.batch_size)

# True if you want to adopt SRB-inspired loss function
MODEL_TYPE = args.model_type

if mode == "test":
    mode_char = "L"
else:
    mode_char = "B"

if not TRAIN:
    file_name = "pls{}_10k".format(DIM)
else:
    file_name = "pls{}_{}".format(DIM, args.num_sol)

VAL_SIZE = args.validation_size

NUM_SOL = args.num_sol

# Model name for both training and test
model_name = "test-{}/".format(TEST_NUM)

# Where to save plots
SAVE_PATH = "plots/test-{}/".format(TEST_NUM)
try:
    os.makedirs(SAVE_PATH)
except:
    print("Directory already exists")

# Model name for both training and test
model_name = "test-{}/".format(TEST_NUM)

# Where to save plots
SAVE_PATH = "plots/test-{}/".format(TEST_NUM)
try:
    os.makedirs(SAVE_PATH)
except:
    print("Directory already exists")

########################################################################################################################

# Create a validation set if required
val_indexes = None

if VAL_SIZE > 0:
    print("Loading validation set...")
    start = time.time()
    X_val = pd.read_csv("datasets/pls{}/partial_solutions_{}_train.csv".format(DIM, NUM_SOL),
                        sep=',',
                        header=None,
                        nrows=MAX_SIZE,
                        dtype=np.int8).values

    # Create penalties for the examples
    if MODEL_TYPE != 'agnostic':
        P_val = pd.read_csv("datasets/pls{}/domains_train_{}.csv".format(DIM, NUM_SOL),
                            sep=',',
                            header=None,
                            nrows=MAX_SIZE,
                            dtype=np.int8).values
    else:
        P_val = np.zeros_like(X_val, dtype=np.int8)

    end = time.time()
    print("Elapsed {} seconds".format((end - start)))

    val_indexes = np.random.choice(np.arange(0, X_val.shape[0]), size=VAL_SIZE, replace=False)
    X_val = X_val[val_indexes]
    P_val = P_val[val_indexes]

    # NOTE: if the model architecture is convolutional then we switch from the one-hot encoding to a 2D dimensional
    #  representation
    if args.model == 'cnn':
        X_val = from_one_hot_to_2d(flattened_array=X_val)
        # We also add the fake channel dimension
        X_val = np.expand_dims(X_val, axis=-1)

    validation_set = (X_val, P_val)

# Load training examples
features_filepath = "datasets/pls{}/partial_solutions_{}_{}.csv".format(DIM, NUM_SOL, mode)
print("Loading features from {}...".format(features_filepath))
start = time.time()
X = pd.read_csv(features_filepath, sep=',', header=None, nrows=MAX_SIZE, dtype=np.int8).values
end = time.time()
print("Elapsed {} seconds, {} GB required".format((end - start), X.nbytes / 10 ** 9))
print("Number of rows: {}".format(X.shape[0]))

# NOTE: if the model architecture is convolutional then we switch from the one-hot encoding to a 2D dimensional
#  representation
if args.model == 'cnn':
    X = from_one_hot_to_2d(flattened_array=X)
    # We also add the fake channel dimension
    X = np.expand_dims(X, axis=-1)

labels_filepath = "datasets/pls{}/assignments_{}_{}.csv".format(DIM, NUM_SOL, mode)
print("Loading labels from {}...".format(labels_filepath))
start = time.time()
Y = pd.read_csv(labels_filepath, sep=',', header=None, nrows=MAX_SIZE, dtype=np.int32).values
end = time.time()
print("Elapsed {} seconds, {} GB required".format((end - start), Y.nbytes / 10 ** 9))

# Create penalties for the examples
if MODEL_TYPE == 'agnostic' and not args.use_prop:
    P = np.zeros((len(X), DIM**3), dtype=np.int8)
else:
    if not args.leave_columns_domains:
        penalties_filepath = "datasets/pls{}/domains_{}_{}.csv".format(DIM, mode, NUM_SOL)
    else:
        penalties_filepath = "datasets/pls{}/rows_propagation_domains_{}_{}.csv".format(DIM, mode, NUM_SOL)

    print("Loading penalties from {}...".format(penalties_filepath))
    start = time.time()
    P = pd.read_csv(penalties_filepath, sep=',', header=None, nrows=MAX_SIZE, dtype=np.int8).values
end = time.time()
print("Elapsed {} seconds, {} GB required".format((end - start), P.nbytes / 10 ** 9))

# Remove validation samples from the training set
if val_indexes is not None:
    X = np.delete(X, val_indexes, axis=0)
    Y = np.delete(Y, val_indexes, axis=0)
    P = np.delete(P, val_indexes, axis=0)

# Create TF datasets
dataset = tf.data.Dataset.from_tensor_slices((X, Y, P)).shuffle(10000).batch(BATCH_SIZE)

# Create the model
if args.model == 'fnn':
    layers = [
        Dense(input_shape=X.shape[1:],
              units=16,
              activation='relu'),
        Dense(units=16,
              activation='relu')
    ]
elif args.model == 'cnn':
    layers = [
        Conv2D(input_shape=X.shape[1:],
               filters=16,
               kernel_size=(3, 3),
               activation='relu'),
        Flatten(),
        Dense(units=16,
              activation='relu')
    ]
else:
    raise Exception("Model type not vali")

model = MyModel(hidden_layers=layers,
                output_dim=DIM ** 3,
                method=MODEL_TYPE,
                lmbd=args.lmbd)

# Train model
if TRAIN:
    history = model.train(EPOCHS,
                          dataset,
                          "models/{}".format(model_name),
                          DIM,
                          validation_set,
                          args.use_prop,
                          args.patience)

    for name in history.keys():
        values = history[name]

        plt.plot(np.arange(0, len(values)), values,
                 label=name)
        plt.ylim(bottom=0)
        plt.legend()
        plt.savefig("{}/{}.png".format(SAVE_PATH, name))
        plt.close()

        with open("{}/{}.csv".format(SAVE_PATH, name), "w") as file:
            wr = csv.writer(file)
            wr.writerow(values)
            file.close()
    exit(0)

else:
    model.model = tf.saved_model.load("models/{}".format(model_name))

################################################################################

# Test the model

# Make predictions
tensor_X = X.astype(np.float32)
predict_val = tf.nn.softmax(model.model(tensor_X)).numpy()

# Prune values according to constraints propagator if required
if args.use_prop:
    predict_val *= (1 - P)

# Count of correct predictions grouped by number of assigned variables
pred_by_num_assigned = np.zeros(shape=(DIM ** 2))
# Count of feasible solutions grouped by number of assigned variables
feas_by_num_assigned = np.zeros(shape=(DIM ** 2))
# Count of total examples grouped by number of assigned variables
tot_by_num_assigned = np.zeros(shape=(DIM ** 2))
# Count of random correct predictions grouped by number of assigned variables
rand_pred_by_num_assigned = np.zeros(shape=(DIM ** 2))
# Count of random feasible solutions grouped by number of assigned variables
rand_feas_by_num_assigned = np.zeros(shape=(DIM ** 2))

# Compute overall accuracy on training set
acc = 0
count = 0
acc_rand = 0

# Compute accuracy grouped by number of assigned variables
preds = []
for x, pred, y, d in zip(X, predict_val, Y, P):

    # NOTE: if the model is convolutional then get the input back to the flattened one-hot representation
    if args.model == 'cnn':
        x = from_2d_to_one_hot(x, dim=DIM)

    if count % 1000 == 0:
        print("Examined {} instances".format(count))

    num_assigned_vars = np.sum(x.astype(np.int8))
    pred_label = np.argmax(pred.reshape(-1))
    correct_label = np.argmax(y.reshape(-1))

    if pred_label == correct_label:
        acc += 1
        pred_by_num_assigned[num_assigned_vars] += 1

    # Create a problem instance with current examples for net prediction
    square = np.reshape(x, (DIM, DIM, DIM))
    pls = PLSInstance(n=DIM)
    pls.square = square.copy()
    # assert pls.__check_constraints__(), "Constraints should be verified before assignment"

    # Make the prediction assignment
    assignment = np.argmax(pred)
    assignment = np.unravel_index(assignment, shape=(DIM, DIM, DIM))

    # Local consistency
    local_feas = pls.assign(assignment[0], assignment[1], assignment[2])

    '''vals_square = np.argmax(square, axis=2) + np.sum(square, axis=2)
    solver = utility.PLSSolver(DIM, square=np.reshape(vals_square, -1))
    res = solver.solve()
    assert res, "Constraint solver is wrong because the input comes from a real solution"'''

    # Global consistency
    if local_feas:
        vals_square = np.argmax(pls.square.copy(), axis=2) + np.sum(pls.square.copy(), axis=2)
        solver = PLSSolver(DIM, square=np.reshape(vals_square, -1))
        feas = solver.solve()
    else:
        feas = local_feas

    if feas:
        feas_by_num_assigned[num_assigned_vars] += 1

    # Check random assignment performance if required
    if args.rnd_feas:
        if not args.use_prop:
            d = None
        rand_assignment = random_assigner(DIM ** 3, d)
        if rand_assignment == correct_label:
            acc_rand += 1
            rand_pred_by_num_assigned[num_assigned_vars] += 1

        # Create a problem instance with current training example for random prediction
        square = np.reshape(x, (DIM, DIM, DIM))
        pls = PLSInstance(n=DIM)
        pls.square = square.copy()
        # assert pls.__check_constraints__(), "Constraints should be verified before assignment"

        # Make the random assignment
        rand_assignment = np.unravel_index(rand_assignment, shape=(DIM, DIM, DIM))

        local_feas = pls.assign(rand_assignment[0], rand_assignment[1], rand_assignment[2])

        # Check global consistency
        if local_feas:
            vals_square = np.argmax(pls.square.copy(), axis=2) + np.sum(pls.square.copy(), axis=2)
            solver = PLSSolver(DIM, square=np.reshape(vals_square, -1))
            feas = solver.solve()
        else:
            feas = local_feas

        if feas:
            rand_feas_by_num_assigned[num_assigned_vars] += 1

    # Increase count of solutions with this number of assignments
    tot_by_num_assigned[num_assigned_vars] += 1
    count += 1

    # Save results checkpoint
    if count % 1000 == 0:

        feasibility = list((feas_by_num_assigned / (tot_by_num_assigned + 1e-8))[1:])

        if not args.use_prop:
            filename = "{}/feasibility_{}.csv".format(SAVE_PATH, mode)
        else:
            if args.leave_columns_domains:
                filename = "{}/feasibility_{}_with_row_prop.csv".format(SAVE_PATH, mode)
            else:
                filename = "{}/feasibility_{}_with_full_prop.csv".format(SAVE_PATH, mode)

        with open(filename, "w") as epoch_file:
            wr = csv.writer(epoch_file)
            wr.writerow(feasibility)

# Check accuracy is correctly computed
assert np.sum(pred_by_num_assigned) == acc and np.sum(tot_by_num_assigned) == count, \
    "acc: {} | acc_vectorized: {} | count: {} | count_vectorized: {}".format(acc, np.sum(pred_by_num_assigned),
                                                                             count, np.sum(tot_by_num_assigned))

# Make plots

accuracy = list((pred_by_num_assigned / (tot_by_num_assigned + 1e-8))[1:])
feasibility = list((feas_by_num_assigned / (tot_by_num_assigned + 1e-8))[1:])
if args.rnd_feas:
    random_feasibility = list((rand_feas_by_num_assigned / (tot_by_num_assigned + 1e-8))[1:])

# Save random assigner results
if args.rnd_feas:
    RANDOM_SAVE_PATH = "plots/test-pls-{}-tf-keras/random/".format(DIM)

    if args.use_prop:
        if not args.leave_columns_domains:
            RANDOM_SAVE_PATH += "rows-and-columns-prop"
        else:
            RANDOM_SAVE_PATH += "rows-prop"
    else:
        RANDOM_SAVE_PATH += "no-prop"

    try:
        os.makedirs(RANDOM_SAVE_PATH)
    except:
        print("Directory {} already exists".format(RANDOM_SAVE_PATH))

    with open("{}/random_feasibility.csv".format(RANDOM_SAVE_PATH, mode), "w") as epoch_file:
        wr = csv.writer(epoch_file)
        wr.writerow(random_feasibility)


import numpy as np
import random
import cv2
import tensorflow as tf
from tensorflow.contrib import slim

def resize_image(image, min_dim=None, max_dim=None, min_scale=None, mode="square"):

    # Keep track of image dtype and return results in the same dtype
    image_dtype = image.dtype
    # Default window (y1, x1, y2, x2) and default scale == 1.
    h, w = image.shape[:2]
    window = (0, 0, h, w)
    scale = 1
    padding = [(0, 0), (0, 0), (0, 0)]
    crop = None

    if mode == "none":
        return image, window, scale, padding, crop

    # Does it exceed max dim?
    if max_dim and mode == "square":
        image_max = max(h, w)

        scale = max_dim / image_max

    # Resize image using bilinear interpolation

    if scale != 1:
        image = cv2.resize(image, dsize=(round(w * scale), round(h * scale)),interpolation=cv2.INTER_AREA)

    # Need padding or cropping?
    if mode == "square":
        # Get new height and width
        h, w = image.shape[:2]
        top_pad = (max_dim - h) // 2
        bottom_pad = max_dim - h - top_pad
        left_pad = (max_dim - w) // 2
        right_pad = max_dim - w - left_pad
        padding = [(top_pad, bottom_pad), (left_pad, right_pad), (0, 0)]
        image = np.pad(image, padding, mode='constant', constant_values=0)
        window = (top_pad, left_pad, h + top_pad, w + left_pad)
    elif mode == "pad64":
        h, w = image.shape[:2]
        # Both sides must be divisible by 64
        assert min_dim % 64 == 0, "Minimum dimension must be a multiple of 64"
        # Height
        if h % 64 > 0:
            max_h = h - (h % 64) + 64
            top_pad = (max_h - h) // 2
            bottom_pad = max_h - h - top_pad
        else:
            top_pad = bottom_pad = 0
        # Width
        if w % 64 > 0:
            max_w = w - (w % 64) + 64
            left_pad = (max_w - w) // 2
            right_pad = max_w - w - left_pad
        else:
            left_pad = right_pad = 0
        padding = [(top_pad, bottom_pad), (left_pad, right_pad), (0, 0)]
        image = np.pad(image, padding, mode='constant', constant_values=0)
        window = (top_pad, left_pad, h + top_pad, w + left_pad)
    elif mode == "crop":
        # Pick a random crop
        h, w = image.shape[:2]
        y = random.randint(0, (h - min_dim))
        x = random.randint(0, (w - min_dim))
        crop = (y, x, min_dim, min_dim)
        image = image[y:y + min_dim, x:x + min_dim]
        window = (0, 0, min_dim, min_dim)
    else:
        raise Exception("Mode {} not supported".format(mode))
    return image.astype(image_dtype), window, scale, padding, crop

def compute_iou(box, boxes, box_area, boxes_area):
    """Calculates IoU of the given box with the array of the given boxes.
    box: 1D vector [y1, x1, y2, x2]
    boxes: [boxes_count, (y1, x1, y2, x2)]
    box_area: float. the area of 'box'
    boxes_area: array of length boxes_count.

    Note: the areas are passed in rather than calculated here for
          efficency. Calculate once in the caller to avoid duplicate work.
    """
    # Calculate intersection areas
    y1 = np.maximum(box[0], boxes[:, 0])
    y2 = np.minimum(box[2], boxes[:, 2])
    x1 = np.maximum(box[1], boxes[:, 1])
    x2 = np.minimum(box[3], boxes[:, 3])
    intersection = np.maximum(x2 - x1, 0) * np.maximum(y2 - y1, 0)
    union = box_area + boxes_area[:] - intersection[:]
    iou = intersection / union
    return iou

def compute_overlaps(boxes1, boxes2):
    """Computes IoU overlaps between two sets of boxes.
    boxes1, boxes2: [N, (y1, x1, y2, x2)].

    For better performance, pass the largest set first and the smaller second.
    """
    # Areas of anchors and GT boxes
    area1 = (boxes1[:, 2] - boxes1[:, 0]) * (boxes1[:, 3] - boxes1[:, 1])
    area2 = (boxes2[:, 2] - boxes2[:, 0]) * (boxes2[:, 3] - boxes2[:, 1])

    # Compute overlaps to generate matrix [boxes1 count, boxes2 count]
    # Each cell contains the IoU value.
    overlaps = np.zeros((boxes1.shape[0], boxes2.shape[0]))
    for i in range(overlaps.shape[1]):
        box2 = boxes2[i]
        overlaps[:, i] = compute_iou(box2, boxes1, area2[i], area1)
    return overlaps

def norm_boxes(boxes, shape):
    """Converts boxes from pixel coordinates to normalized coordinates.
    boxes: [N, (y1, x1, y2, x2)] in pixel coordinates
    shape: [..., (height, width)] in pixels

    Note: In pixel coordinates (y2, x2) is outside the box. But in normalized
    coordinates it's inside the box.

    Returns:
        [N, (y1, x1, y2, x2)] in normalized coordinates
    """
    h, w = shape
    scale = np.array([h - 1, w - 1, h - 1, w - 1])
    shift = np.array([0, 0, 1, 1])

    return np.divide((boxes - shift), scale).astype(np.float32)

def apply_box_deltas_graph(boxes, deltas):
    """Applies the given deltas to the given boxes.
    boxes: [N, (y1, x1, y2, x2)] boxes to update
    deltas: [N, (dy, dx, log(dh), log(dw))] refinements to apply
    """
    # Convert to y, x, h, w
    width = boxes[:, 3] - boxes[:, 1]
    height = boxes[:, 2] - boxes[:, 0]
    center_y = boxes[:, 0] + 0.5 * height
    center_x = boxes[:, 1] + 0.5 * width
    # Apply deltas
    center_y += deltas[:, 0] * height
    center_x += deltas[:, 1] * width
    height *= tf.exp(deltas[:, 2])
    width *= tf.exp(deltas[:, 3])
    # Convert back to y1, x1, y2, x2
    y1 = center_y - 0.5 * height
    x1 = center_x - 0.5 * width
    y2 = y1 + height
    x2 = x1 + width
    result = tf.stack([y1, x1, y2, x2], axis=1, name="apply_box_deltas_out")
    return result

def clip_boxes_graph(boxes, window):
    """
    boxes: [N, (y1, x1, y2, x2)]
    window: [4] in the form y1, x1, y2, x2
    """
    # Split
    wy1, wx1, wy2, wx2 = tf.split(window, 4)
    y1, x1, y2, x2 = tf.split(boxes, 4, axis=1)
    # Clip
    y1 = tf.maximum(tf.minimum(y1, wy2), wy1)
    x1 = tf.maximum(tf.minimum(x1, wx2), wx1)
    y2 = tf.maximum(tf.minimum(y2, wy2), wy1)
    x2 = tf.maximum(tf.minimum(x2, wx2), wx1)
    clipped = tf.concat([y1,x1, y2, x2], axis=1, name="clipped_boxes")
    clipped.set_shape((clipped.shape[0], 4))
    return clipped

def nms(boxes, scores,cfg):
    indices = tf.image.non_max_suppression(
        boxes, scores, cfg.NMS_ROIS_TRAINING,
        cfg.RPN_NMS_THRESHOLD, name="rpn_non_max_suppression")
    proposals = tf.gather(boxes, indices)

    padding = tf.maximum(cfg.NMS_ROIS_TRAINING - tf.shape(proposals)[0], 0)
    proposals = tf.pad(proposals, [(0, padding), (0, 0)])
    return proposals
def trim_zeros_graph(boxes, name=None):
    """Often boxes are represented with matricies of shape [N, 4] and
    are padded with zeros. This removes zero boxes.

    boxes: [N, 4] matrix of boxes.
    non_zeros: [N] a 1D boolean mask identifying the rows to keep
    """
    non_zeros = tf.cast(tf.reduce_sum(tf.abs(boxes), axis=1), tf.bool)
    boxes = tf.boolean_mask(boxes, non_zeros, name=name)
    return boxes, non_zeros

def trim_zeros(boxes, name=None):
    """Often boxes are represented with matricies of shape [N, 4] and
    are padded with zeros. This removes zero boxes.

    boxes: [N, 4] matrix of boxes.
    non_zeros: [N] a 1D boolean mask identifying the rows to keep
    """
    non_zeros =tf.reduce_sum(boxes, axis=1)>0
    boxes = tf.boolean_mask(boxes, non_zeros, name=name)
    return boxes, non_zeros

def overlaps_graph(boxes1, boxes2):
    """Computes IoU overlaps between two sets of boxes.
    boxes1, boxes2: [N, (y1, x1, y2, x2)].
    """
    # 1. Tile boxes2 and repeate boxes1. This allows us to compare
    # every boxes1 against every boxes2 without loops.
    # TF doesn't have an equivalent to np.repeate() so simulate it
    # using tf.tile() and tf.reshape.
    b1 = tf.reshape(tf.tile(tf.expand_dims(boxes1, 1),
                            [1, 1, tf.shape(boxes2)[0]]), [-1, 4])
    b2 = tf.tile(boxes2, [tf.shape(boxes1)[0], 1])
    # 2. Compute intersections
    b1_y1, b1_x1, b1_y2, b1_x2 = tf.split(b1, 4, axis=1)
    b2_y1, b2_x1, b2_y2, b2_x2 = tf.split(b2, 4, axis=1)
    y1 = tf.maximum(b1_y1, b2_y1)
    x1 = tf.maximum(b1_x1, b2_x1)
    y2 = tf.minimum(b1_y2, b2_y2)
    x2 = tf.minimum(b1_x2, b2_x2)
    intersection = tf.maximum(x2 - x1, 0) * tf.maximum(y2 - y1, 0)
    # 3. Compute unions
    b1_area = (b1_y2 - b1_y1) * (b1_x2 - b1_x1)
    b2_area = (b2_y2 - b2_y1) * (b2_x2 - b2_x1)
    union = b1_area + b2_area - intersection
    # 4. Compute IoU and reshape to [boxes1, boxes2]
    iou = intersection / union
    overlaps = tf.reshape(iou, [tf.shape(boxes1)[0], tf.shape(boxes2)[0]])
    return overlaps

def box_refinement_graph(box, gt_box):
    """Compute refinement needed to transform box to gt_box.
    box and gt_box are [N, (y1, x1, y2, x2)]
    """
    box = tf.cast(box, tf.float32)
    gt_box = tf.cast(gt_box, tf.float32)

    height = box[:, 2] - box[:, 0]
    width = box[:, 3] - box[:, 1]
    center_y = box[:, 0] + 0.5 * height
    center_x = box[:, 1] + 0.5 * width

    gt_height = gt_box[:, 2] - gt_box[:, 0]
    gt_width = gt_box[:, 3] - gt_box[:, 1]
    gt_center_y = gt_box[:, 0] + 0.5 * gt_height
    gt_center_x = gt_box[:, 1] + 0.5 * gt_width

    dy = (gt_center_y - center_y) / height
    dx = (gt_center_x - center_x) / width
    dh = tf.log(gt_height / height)
    dw = tf.log(gt_width / width)

    result = tf.stack([dy, dx, dh, dw], axis=1)
    return result
def log2_graph(x):

    return tf.log(x) / tf.log(2.0)

def roi_align(boxes,feature_maps,cfg):

    # Assign each ROI to a level in the pyramid based on the ROI area.
    y1, x1, y2, x2 = tf.split(boxes, 4, axis=2)
    h = y2 - y1
    w = x2 - x1
    # Use shape of first image. Images in a batch must have the same size.
    image_shape = cfg.image_size
    # Equation 1 in the Feature Pyramid Networks paper. Account for
    # the fact that our coordinates are normalized here.
    # e.g. a 224x224 ROI (in pixels) maps to P4
    image_area = tf.cast(image_shape[0] * image_shape[1], tf.float32)
    roi_level = log2_graph(tf.sqrt(h * w) / (224.0 / tf.sqrt(image_area)))

    roi_level = tf.minimum(2, tf.maximum(
        0, 1 + tf.cast(tf.round(roi_level), tf.int32)))
    roi_level = tf.squeeze(roi_level, 2)

    # Loop through levels and apply ROI pooling to each. P2 to P5.
    pooled = []
    box_to_level = []
    for i, level in enumerate(range(3)):
        ix = tf.where(tf.equal(roi_level, level))
        level_boxes = tf.gather_nd(boxes, ix)

        # Box indicies for crop_and_resize.
        box_indices = tf.cast(ix[:, 0], tf.int32)

        # Keep track of which box is mapped to which level
        box_to_level.append(ix)

        # Stop gradient propogation to ROI proposals
        level_boxes = tf.stop_gradient(level_boxes)
        box_indices = tf.stop_gradient(box_indices)

        # Crop and Resize
        # From Mask R-CNN paper: "We sample four regular locations, so
        # that we can evaluate either max or average pooling. In fact,
        # interpolating only a single value at each bin center (without
        # pooling) is nearly as effective."
        #
        # Here we use the simplified approach of a single value per bin,
        # which is how it's done in tf.crop_and_resize()
        # Result: [batch * num_boxes, pool_height, pool_width, channels]
        pooled.append(tf.image.crop_and_resize(
            feature_maps[i], level_boxes, box_indices, [cfg.pool_shape,cfg.pool_shape],
            method="bilinear"))

    # Pack pooled features into one tensor
    pooled = tf.concat(pooled, axis=0)

    # Pack box_to_level mapping into one array and add another
    # column representing the order of pooled boxes
    box_to_level = tf.concat(box_to_level, axis=0)
    box_range = tf.expand_dims(tf.range(tf.shape(box_to_level)[0]), 1)
    box_to_level = tf.concat([tf.cast(box_to_level, tf.int32), box_range],
                             axis=1)

    # Rearrange pooled features to match the order of the original boxes
    # Sort box_to_level by batch then box index
    # TF doesn't have a way to sort by two columns, so merge them and sort.
    sorting_tensor = box_to_level[:, 0] * 100000 + box_to_level[:, 1]
    ix = tf.nn.top_k(sorting_tensor, k=tf.shape(
        box_to_level)[0]).indices[::-1]
    ix = tf.gather(box_to_level[:, 2], ix)
    pooled = tf.gather(pooled, ix)

    # Re-add the batch dimension
    #pooled = tf.expand_dims(pooled, 0)
    return pooled

def batch_pack_graph(x, counts, num_rows):
    """Picks different number of values from each row
    in x depending on the values in counts.
    """
    outputs = []
    for i in range(num_rows):
        outputs.append(x[i, 0:counts[i],:])
    return tf.concat(outputs, axis=0)

def refine_detections_graph(rois, probs, deltas,window,cfg):
    """Refine classified proposals and filter overlaps and return final
    detections.

    Inputs:
        rois: [N, (y1, x1, y2, x2)] in normalized coordinates
        probs: [N, num_classes]. Class probabilities.
        deltas: [N, num_classes, (dy, dx, log(dh), log(dw))]. Class-specific
                bounding box deltas.
        window: (y1, x1, y2, x2) in image coordinates. The part of the image
            that contains the image excluding the padding.

    Returns detections shaped: [N, (y1, x1, y2, x2, class_id, score)] where
        coordinates are normalized.
    """
    # Class IDs per ROI
    probs = tf.squeeze(probs,axis=1)
    probs = tf.squeeze(probs, axis=1)
    rois = tf.squeeze(rois,axis=0)
    class_ids = tf.argmax(probs, axis=1, output_type=tf.int32)
    class_scores = tf.reduce_max(probs, axis=1)
    # Class probability of the top class of each ROI

    ix = tf.range(cfg.NMS_ROIS_TRAINING)
    idece = tf.stack([ix,class_ids],axis=1)

    deltas_specific = tf.gather_nd(deltas,idece)

    refined_rois = apply_box_deltas_graph(
        rois, deltas_specific * cfg.BBOX_STD_DEV)
    # Clip boxes to image window
    refined_rois = clip_boxes_graph(refined_rois, window)

    # TODO: Filter out boxes with zero area

    # Filter out background boxes
    keep = tf.where(class_ids > 0)[:, 0]
    # Filter out low confidence boxes
    if cfg.DETECTION_MIN_CONFIDENCE:
        conf_keep = tf.where(class_scores >= cfg.DETECTION_MIN_CONFIDENCE)[:, 0]
        keep = tf.sets.set_intersection(tf.expand_dims(keep, 0),
                                        tf.expand_dims(conf_keep, 0))
        keep = tf.sparse_tensor_to_dense(keep)[0]

    # Apply per-class NMS
    # 1. Prepare variables
    pre_nms_class_ids = tf.gather(class_ids, keep)
    pre_nms_scores = tf.gather(class_scores, keep)
    pre_nms_rois = tf.gather(refined_rois,   keep)
    unique_pre_nms_class_ids = tf.unique(pre_nms_class_ids)[0]

    def nms_keep_map(class_id):
        """Apply Non-Maximum Suppression on ROIs of the given class."""
        # Indices of ROIs of the given class
        ixs = tf.where(tf.equal(pre_nms_class_ids, class_id))[:, 0]
        # Apply NMS
        class_keep = tf.image.non_max_suppression(
                tf.gather(pre_nms_rois, ixs),
                tf.gather(pre_nms_scores, ixs),
                max_output_size=cfg.DETECTION_MAX_INSTANCES,
                iou_threshold=cfg.DETECTION_NMS_THRESHOLD)
        # Map indicies
        class_keep = tf.gather(keep, tf.gather(ixs, class_keep))
        # Pad with -1 so returned tensors have the same shape
        gap = cfg.DETECTION_MAX_INSTANCES - tf.shape(class_keep)[0]
        class_keep = tf.pad(class_keep, [(0, gap)],
                            mode='CONSTANT', constant_values=-1)
        # Set shape so map_fn() can infer result shape
        class_keep.set_shape([cfg.DETECTION_MAX_INSTANCES])
        return class_keep

    # 2. Map over class IDs
    nms_keep = tf.map_fn(nms_keep_map, unique_pre_nms_class_ids,
                         dtype=tf.int64)
    # 3. Merge results into one list, and remove -1 padding
    nms_keep = tf.reshape(nms_keep, [-1])
    nms_keep = tf.gather(nms_keep, tf.where(nms_keep > -1)[:, 0])

    # 4. Compute intersection between keep and nms_keep
    keep = tf.sets.set_intersection(tf.expand_dims(keep, 0),
                                    tf.expand_dims(nms_keep, 0))
    keep = tf.sparse_tensor_to_dense(keep)[0]
    # Keep top detections
    roi_count = cfg.DETECTION_MAX_INSTANCES
    class_scores_keep = tf.gather(class_scores, keep)
    num_keep = tf.minimum(tf.shape(class_scores_keep)[0], roi_count)
    top_ids = tf.nn.top_k(class_scores_keep, k=num_keep, sorted=True)[1]
    keep = tf.gather(keep, top_ids)

    # Arrange output as [N, (y1, x1, y2, x2, class_id, score)]
    # Coordinates are normalized.
    detections = tf.concat([
        tf.gather(refined_rois, keep),
        tf.to_float(tf.gather(class_ids, keep))[..., tf.newaxis],
        tf.gather(class_scores, keep)[..., tf.newaxis]
        ], axis=1)

    # Pad with zeros if detections < DETECTION_MAX_INSTANCES
    gap = cfg.DETECTION_MAX_INSTANCES - tf.shape(detections)[0]
    detections = tf.pad(detections, [(0, gap), (0, 0)], "CONSTANT")
    return detections

def faster_rcnn_arg(weight_decay=0.00004,
                                  batch_norm_decay=0.9997,
                                  batch_norm_epsilon=0.001,
                                  activation_fn=tf.nn.relu):
  """Returns the scope with the default parameters for inception_resnet_v2.

  Args:
    weight_decay: the weight decay for weights variables.
    batch_norm_decay: decay for the moving average of batch_norm momentums.
    batch_norm_epsilon: small float added to variance to avoid dividing by zero.
    activation_fn: Activation function for conv2d.

  Returns:
    a arg_scope with the parameters needed for inception_resnet_v2.
  """
  # Set weight_decay for weights in conv2d and fully_connected layers.
  with slim.arg_scope([slim.conv2d, slim.fully_connected],
                      weights_regularizer=slim.l2_regularizer(weight_decay),
                      biases_regularizer=slim.l2_regularizer(weight_decay)):

    batch_norm_params = {
        'decay': batch_norm_decay,
        'epsilon': batch_norm_epsilon,
        'fused': None,  # Use fused batch norm if possible.
    }
    # Set activation_fn and parameters for batch_norm.
    with slim.arg_scope([slim.conv2d], activation_fn=activation_fn,
                        normalizer_fn=slim.batch_norm,
                        normalizer_params=batch_norm_params) as scope:
      return scope


"""
Программа: Сохранение моделей и их параметров
Версия: 1.0
"""

import joblib
import pandas as pd
import yaml
from lightgbm import LGBMClassifier
from catboost import CatBoostClassifier

def load_recommend_model(model: LGBMClassifier, supplier_id: int, **kwargs) -> None:
    """
    Cораняет обученную рекомендательную модель и ее параметры для конкретного поставщика.

    Аргументы:
        model (LGBMClassifier): обученная модель
        supplier_id (int): идентификатор поставщика.
        kwargs (dict):
            models (str): путь к файлу конфигурации, содержащему модели.
            params (str): путь к файлу конфигурации, содержащему параметы моделей.
    """
    # загрузка моделей и их параметров
    with open(kwargs['params']) as file:
        params = yaml.load(file, Loader=yaml.FullLoader)
    models = joblib.load(kwargs['models'])

    # загружаем модель и параметры в словарь по индексу supplier_id
    models[supplier_id] = model
    params[supplier_id] = model.get_params()

    # сохраняем словарь моделей и параметров
    joblib.dump(models, kwargs['models'])
    with open(kwargs['params'], 'w') as f:
        yaml.dump(params, f)

def load_model(model: CatBoostClassifier, **kwargs) -> None:
    """
    Cохраняет обученную модель, предсказывающую победителя, и ее параметры.

    Аргументы:
        model (СatBoostClassifier): обученная модель
        kwargs (dict):
            models (str): путь к файлу конфигурации, содержащему модели.
            params (str): путь к файлу конфигурации, содержащему параметы моделей.
    """
    # загрузка моделей и их параметров
    with open(kwargs['params']) as file:
        params = yaml.load(file, Loader=yaml.FullLoader)
    models = joblib.load(kwargs['models'])

    # загружаем модель catboost и ее параметры
    models['catboost'] = model
    params['catboost'] = model.get_params()

    # сохраняем словарь моделей и параметров
    joblib.dump(models, kwargs['models'])
    with open(kwargs['params'], 'w') as f:
        yaml.dump(params, f)

def load_data(df_train: pd.DataFrame, df_test: pd.DataFrame, df_recommend_submission: pd.DataFrame,
              df_evaluate: pd.DataFrame, **kwargs) -> None:
    """
    Загрузка данных из файла конфигурации и сохранение обработанных данных в CSV-файлы.

    Аргументы:
        df_train: DataFrame с тренировочными данными.
        df_test: DataFrame с тестовыми данными.
        df_recommend_submission: DataFrame с данными для получения предсказаний.
        df_evaluate: DataFrame с данными для оценки модели.
        kwargs: Аргументы для функции. Содержит:
            train_data (str): Путь тренировочных предобработанных данных.
            test_data (str): Путь тестовых предобработанных данных.
            recommend_sub_path (str): Путь файла, содержащего поставщиков и их закупке из тестовых данных
            evaluate_data (str): Путь данных для получения предсказаний
    """

    # Сохранение данных в CSV-файлы
    df_train.to_csv(kwargs['train_data'], index_label='index')
    df_test.to_csv(kwargs['test_data'], index_label='index')
    df_recommend_submission.to_csv(kwargs['recommend_sub_path'], index_label='index')
    df_evaluate.to_csv(kwargs['evaluate_data'], index_label='index')


# -*- coding:utf-8 -*-
'''
Created on 2018年2月28日
@author: ning.lin
'''
'''
大图地址class或id有big字样 的

小图地址

'''
# url="http://search.jiayuan.com/v2/search_v2.php"

import json
import urllib.parse

import requests

# url='http://sou.zhaopin.com/jobs/searchresult.ashx?jl=%E5%B9%BF%E5%B7%9E&kw=%E8%BF%90%E7%BB%B4&p=1&isadv=0'
url = 'http://search.jiayuan.com/v2/index.php?key=&sex=f&stc=&sn=default&sv=1&p=1&pt=1454903&ft=off&f=select&mt=d'
url = 'http://search.jiayuan.com/v2/search_v2.php'
a = '%E8%BF%90%E7%BB%B4'
b='喂,要幸福\x0e_33岁_32595588'
c='\u3000\u3000'
print("aa",urllib.parse.unquote(c))
data = {
    'sex': 'f',
    'key':'',
    'stc':'',
    'sn': 'default',
    'sv': 2,
    'p': 2,
    'f': 'select',
    'listStyle': 'bigPhoto',
    'pri_uid':'',
    'jsversion': 'v5',
}
# 1
# requests.post(url, data=json.dumps(data))
req = requests.get(url)

# 2-json参数会自动将字典类型的对象转换为json格式
req = requests.post(url, json=data)
# print(req.text)
a = json.loads(req.text)
print(type(a),a)


import os
import glob
import datetime
import email.parser
import codecs

from .formats import format_for_extension, UnknownExtension

def gather_files(path, recursive=False):
    gathered_files = []

    files = glob.glob(os.path.join(path, '*'))
    while files:
        f = files.pop(0)
        if os.path.isdir(f):
            if recursive:
                files += glob.glob(os.path.join(f, '*'))
        else:
            gathered_files.append(f.replace(path, '').lstrip('/'))

    return gathered_files

def email_to_dict(file_descriptor):
    message = email.parser.Parser().parse(file_descriptor)

    # body
    email_dict = { 'body': [ message.get_payload() ] }

    # headers
    for (key, value) in message.items():
        key = key.lower() # lowercase them all to make for easy comparison
        if key not in email_dict:
            email_dict[key] = []
        email_dict[key].append(value)

    return email_dict

class Parser(object):

    def __init__(self, manager, **global_defaults):
        self.manager = manager
        self.defaults = global_defaults
        self._header_map = {
            'author': 'authors',
            'date': 'created_on',
            'created-date': 'created_on',
            'publish-date': 'publish_on',
        }

    def _create_doc(self, **extra):
        data = dict(self.defaults)
        data.update(extra)
        return self.manager.document_class(**data)

    def map_header_to_field(self, **extra):
        self._header_map.update(extra)

    def parse(self, source_path, recursive=False, **defaults):
        self.parse_folder(source_path, recursive=recursive, **defaults)

    def parse_folder(self, path, recursive=False, **defaults):
        path = os.path.abspath(path)

        # get all the files that are candidates for parsing
        files = gather_files(path, recursive=recursive)

        # filter all the filters that haven't changed
        # files = self.filter_files(path, files, last_run=FOOBAR)

        for file in files:
            self.parse_file(path, file, **defaults)

    def parse_file(self, path, filepath, **defaults):
        doc = self._create_doc(**defaults)
        self.build_document(path, filepath, doc)
        self.manager.save(doc)

    def build_document(self, path, filepath, doc):
        self.load_file_info(path, filepath, doc)
        self.load_file_content(path, filepath, doc)

    def load_file_info(self, path, filepath, doc):
        mtime = os.path.getmtime(os.path.join(path, filepath))
        mod_dt = datetime.datetime.fromtimestamp(mtime)

        (doc_path, ext) = os.path.splitext(filepath)

        doc.path = doc_path
        doc.created_on = mod_dt
        doc.modified_on = mod_dt
        doc.publish_on = mod_dt

        try:
            doc.format = format_for_extension(ext)
        except UnknownExtension:
            pass

    def load_file_content(self, path, filepath, doc):
        with codecs.open(os.path.join(path, filepath), mode='r', encoding='utf-8') as fd:
            data = email_to_dict(fd)

        # Grab any alternate slugs
        alts = data.pop('alt-slug', [])

        # Populate the document
        for (field, values) in data.iteritems():
            for value in values:
                doc.assign(self._header_map.get(field, field), value)

        # Create the redirects
        for alt in alts:
            redirect_doc = doc.create_redirect(alt)
            self.manager.save(redirect_doc)